'
+title: PRQL Compiler
+base: core20 # the base snap is the execution environment for this snap
+version: git
+summary: CLI for PRQL, a modern language for transforming data
+description: |
+ prqlc is the CLI for the PRQL compiler. It compiles PRQL to SQL, and offers various diagnostics.
+
+ PRQL is a modern language for transforming data โ a simple, powerful, pipelined SQL replacement.
+issues: https://github.com/PRQL/prql/issues
+source-code: https://github.com/PRQL/prql
+contact: https://twitter.com/prql_lang
+website: https://prql-lang.org/
+license: Apache-2.0
+grade: devel # must be 'stable' to release into candidate/stable channels
+confinement: strict
+icon: ../web/website/themes/prql-theme/static/icon.svg
+
+parts:
+ prqlc:
+ plugin: rust
+ source: ..
+ rust-path: [prql-compiler/prqlc]
+
+apps:
+ prqlc:
+ command: bin/prqlc
+ plugs:
+ - home
diff --git a/playground/src/sidebar/Sidebar.js b/playground/src/sidebar/Sidebar.js
deleted file mode 100644
index 178c65408509..000000000000
--- a/playground/src/sidebar/Sidebar.js
+++ /dev/null
@@ -1,64 +0,0 @@
-import "./Sidebar.css";
-import React from "react";
-
-function Sidebar({ library, onLoadFile }) {
- function loadFile(section, file) {
- onLoadFile(file, library[section][file]);
- }
-
- const sections = [];
-
- for (const [section, files] of Object.entries(library)) {
- const fileRows = [];
- for (const [index, filename] of Object.keys(files).entries()) {
- fileRows.push(
- loadFile(section, filename)}
- >
- {filename}
-
- );
- }
-
- sections.push(
-
- {section}
-
- {fileRows}
-
- );
- }
-
- return (
-
- );
-}
-
-export default Sidebar;
diff --git a/prql-compiler/Cargo.toml b/prql-compiler/Cargo.toml
index 296cf5b954fe..5cea1a4342f8 100644
--- a/prql-compiler/Cargo.toml
+++ b/prql-compiler/Cargo.toml
@@ -15,34 +15,37 @@ metadata.msrv = "1.65.0"
[dependencies]
anyhow = {version = "1.0.57", features = ["backtrace"]}
ariadne = "0.1.5"
-chumsky = "0.8.0"
csv = "1.2.0"
enum-as-inner = "0.5.0"
itertools = "0.10.3"
lazy_static = "1.4.0"
log = "0.4.17"
once_cell = "1.17.0"
-pest = "2.5.0"
-pest_derive = "2.5.0"
regex = "1.7.0"
semver = {version = "1.0.14", features = ["serde"]}
serde = {version = "1.0.137", features = ["derive"]}
serde_json = "1.0.81"
sqlformat = "0.2.0"
-sqlparser = {version = "0.30.0", features = ["serde"]}
+sqlparser = {version = "0.32.0", features = ["serde"]}
strum = {version = "0.24.0", features = ["std", "derive"]}# for converting enum variants to string
strum_macros = "0.24.0"
+# Chumsky issues in wasm (though we only see it when compiling on MacOS)
+[target.'cfg(not(target_family="wasm"))'.dependencies]
+chumsky = "0.9.2"
+[target.'cfg(target_family="wasm")'.dependencies]
+chumsky = {version = "0.9.2", features = ["ahash", "std"], default-features = false}
+
[dev-dependencies]
cfg-if = "1.0"
insta = {version = "1.28", features = ["colors", "glob", "yaml"]}
# For integration tests
[target.'cfg(not(target_family="wasm"))'.dev-dependencies]
-chrono = "0.4"
-pretty_assertions = "1.3.0"
+chrono = {version = "0.4", features = [], default-features = false}
criterion = "0.4.0"
postgres = "0.19.3"
+pretty_assertions = "1.3.0"
rusqlite = {version = "0.28.0", features = ["bundled", "csvtab"]}
# Re-enable on windows when duckdb supports it
@@ -53,3 +56,11 @@ duckdb = {version = "0.7.0", features = ["bundled", "chrono"]}
[[bench]]
harness = false
name = "bench"
+
+# Putting this in the workspace root causes it to refer to the path relative
+# to `prql-compiler`? We can place it there when that's fixed.
+[[package.metadata.release.pre-release-replacements]]
+exactly = 1
+file = "../web/book/src/language-features/target.md"
+replace = 'prql version:"{{version}}"'
+search = 'prql version:"[\d.]+"'
diff --git a/prql-compiler/README.md b/prql-compiler/README.md
index 900be541dc03..abcdc1202bbe 100644
--- a/prql-compiler/README.md
+++ b/prql-compiler/README.md
@@ -8,7 +8,7 @@ For more on PRQL, check out the [PRQL website](https://prql-lang.org) or the
For more usage examples and the library documentation, check out the
[`prql-compiler` documentation](https://docs.rs/prql-compiler/latest/prql_compiler/).
-# Installation
+## Installation
```shell
cargo add prql-compiler
@@ -24,12 +24,12 @@ Compile a PRQL string to a SQLite dialect string.
use prql_compiler::{compile, Options, Target, sql::Dialect};
let prql = "from employees | select [name, age]";
-let opt = Options {
+let opts = &Options {
format: false,
target: Target::Sql(Some(Dialect::SQLite)),
signature_comment: false
};
-let sql = compile(&prql, opt).unwrap();
+let sql = compile(&prql, opts).unwrap();
assert_eq!("SELECT name, age FROM employees", sql);
```
diff --git a/prql-compiler/benches/bench.rs b/prql-compiler/benches/bench.rs
index 8ab830c42513..a0ec550d6fb5 100644
--- a/prql-compiler/benches/bench.rs
+++ b/prql-compiler/benches/bench.rs
@@ -16,9 +16,9 @@ cfg_if::cfg_if! {
use criterion::{criterion_group, criterion_main, Criterion};
use prql_compiler::*;
- const CONTENT: &str = include_str!("../../book/tests/prql/examples/variables-0.prql");
+ const CONTENT: &str = include_str!("../examples/compile-files/queries/variables.prql");
fn compile_query() -> Result {
- compile(CONTENT, Options::default())
+ compile(CONTENT, &Options::default())
}
fn criterion_benchmark(c: &mut Criterion) {
diff --git a/prql-compiler/examples/compile-files/build.rs b/prql-compiler/examples/compile-files/build.rs
index 6eb9eea9ee68..6edbd2f1d397 100644
--- a/prql-compiler/examples/compile-files/build.rs
+++ b/prql-compiler/examples/compile-files/build.rs
@@ -19,7 +19,7 @@ fn main() {
let prql_string = fs::read_to_string(prql_path).unwrap();
// compile
- let sql_string = compile(&prql_string, Options::default()).unwrap();
+ let sql_string = compile(&prql_string, &Options::default()).unwrap();
// write file
fs::write(sql_path, sql_string).unwrap();
diff --git a/book/tests/prql/examples/variables-0.prql b/prql-compiler/examples/compile-files/queries/variables.prql
similarity index 100%
rename from book/tests/prql/examples/variables-0.prql
rename to prql-compiler/examples/compile-files/queries/variables.prql
diff --git a/prql-compiler/prql-compiler-macros/Cargo.toml b/prql-compiler/prql-compiler-macros/Cargo.toml
index ee28586b7d37..b83c4ea33d55 100644
--- a/prql-compiler/prql-compiler-macros/Cargo.toml
+++ b/prql-compiler/prql-compiler-macros/Cargo.toml
@@ -14,8 +14,10 @@ proc_macro = true
test = false
[dependencies]
-prql-compiler = {path = "..", default-features = false, version = "0.5.1" }
-syn = "1.0"
+prql-compiler = {path = "..", default-features = false, version = "0.6.1" }
+# Was getting build errors with more recent versions; can remove pin if it
+# successfully builds.
+syn = "=1.0.107"
[package.metadata.release]
tag-name = "{{version}}"
diff --git a/prql-compiler/prql-compiler-macros/src/lib.rs b/prql-compiler/prql-compiler-macros/src/lib.rs
index 2e36ebf24760..88e65b2eb281 100644
--- a/prql-compiler/prql-compiler-macros/src/lib.rs
+++ b/prql-compiler/prql-compiler-macros/src/lib.rs
@@ -26,7 +26,7 @@ pub fn prql_to_sql(input: TokenStream) -> TokenStream {
let opts = prql_compiler::Options::default().no_format().no_signature();
- let sql_string = match prql_compiler::compile(&prql_string, opts) {
+ let sql_string = match prql_compiler::compile(&prql_string, &opts) {
Ok(r) => r,
Err(err) => {
panic!("{}", err);
diff --git a/prql-compiler/prqlc/Cargo.toml b/prql-compiler/prqlc/Cargo.toml
index 7a70d90ae406..e95f5a281877 100644
--- a/prql-compiler/prqlc/Cargo.toml
+++ b/prql-compiler/prqlc/Cargo.toml
@@ -1,5 +1,5 @@
[package]
-description = "CLI interface for the PRQL compiler"
+description = "CLI for the PRQL compiler"
name = "prqlc"
edition.workspace = true
@@ -15,10 +15,13 @@ atty = "0.2.14"
clap = {version = "4.1.1", features = ["derive"]}
clio = {version = "0.2.4", features = ['clap-parse']}
color-eyre = "0.6.1"
-env_logger = {version = "0.9.1", features = ["termcolor"]}
+env_logger = {version = "0.10.0", features = ["color"]}
itertools = "0.10.3"
+minijinja = {version = "0.30.4", features = ["unstable_machinery"]}
notify = "^5.1.0"
-prql-compiler = {path = '..', version = "0.5.1" }
+prql-compiler = {path = '..', version = "0.6.1" }
+regex = {version = "1.7.1", features = ["std", "unicode"]}
+serde = "^1"
serde_json = "1.0.81"
serde_yaml = "0.9.1"
walkdir = "^2.3.2"
diff --git a/prql-compiler/prqlc/src/cli.rs b/prql-compiler/prqlc/src/cli.rs
index 555b4c4aa479..41db7dc8825a 100644
--- a/prql-compiler/prqlc/src/cli.rs
+++ b/prql-compiler/prqlc/src/cli.rs
@@ -1,6 +1,6 @@
-use anyhow::{anyhow, Result};
+use anyhow::Result;
use ariadne::Source;
-use clap::Parser;
+use clap::{Parser, Subcommand};
use clio::{Input, Output};
use itertools::Itertools;
use std::io::{Read, Write};
@@ -14,12 +14,13 @@ use prql_compiler::{downcast, Options};
use crate::watch;
+/// Entrypoint called by [crate::main]
pub fn main() -> color_eyre::eyre::Result<()> {
env_logger::builder().format_timestamp(None).init();
color_eyre::install()?;
let mut cli = Cli::parse();
- if let Err(error) = cli.run() {
+ if let Err(error) = cli.command.run() {
eprintln!("{error}");
exit(1)
}
@@ -27,48 +28,73 @@ pub fn main() -> color_eyre::eyre::Result<()> {
Ok(())
}
-#[derive(Parser)]
+#[derive(Parser, Debug, Clone)]
+struct Cli {
+ #[command(subcommand)]
+ command: Command,
+}
+
+#[derive(Subcommand, Debug, Clone)]
#[clap(name = env!("CARGO_PKG_NAME"), about, version)]
-pub enum Cli {
- /// Parse PL AST
- Parse(CommandIO),
+enum Command {
+ /// Parse into PL AST
+ Parse {
+ #[clap(value_parser, default_value = "-")]
+ input: Input,
+ #[clap(value_parser, default_value = "-")]
+ output: Output,
+ #[arg(value_enum, long)]
+ format: Option,
+ },
/// Parse & generate PRQL code back
#[clap(name = "fmt")]
- Format(CommandIO),
+ Format(IoArgs),
/// Parse, resolve & combine source with comments annotating relation type
- Annotate(CommandIO),
+ Annotate(IoArgs),
/// Parse & resolve, but don't lower into RQ
- Debug(CommandIO),
+ Debug(IoArgs),
/// Parse, resolve & lower into RQ
- Resolve(CommandIO),
+ Resolve(IoArgs),
/// Parse, resolve, lower into RQ & compile to SQL
- Compile(CommandIO),
+ Compile(IoArgs),
/// Watch a directory and compile .prql files to .sql files
- Watch(watch::WatchCommand),
+ Watch(watch::WatchArgs),
}
-#[derive(clap::Args, Default)]
-pub struct CommandIO {
+#[derive(clap::Args, Default, Debug, Clone)]
+pub struct IoArgs {
#[clap(value_parser, default_value = "-")]
input: Input,
#[clap(value_parser, default_value = "-")]
output: Output,
+
+ // TODO: This should be only on some commands, is there an elegant way of
+ // doing that in Clap without lots of duplication?
+ #[arg(value_enum, long)]
+ format: Option,
+}
+
+#[derive(clap::ValueEnum, Clone, Debug)]
+enum Format {
+ Json,
+ Yaml,
}
fn is_stdin(input: &Input) -> bool {
input.path() == "-"
}
-impl Cli {
+impl Command {
+ /// Entrypoint called by [`main`]
pub fn run(&mut self) -> Result<()> {
- if let Cli::Watch(command) = self {
+ if let Command::Watch(command) = self {
return watch::run(command);
};
@@ -95,18 +121,16 @@ impl Cli {
fn execute(&self, source: &str) -> Result> {
Ok(match self {
- Cli::Parse(_) => {
- let ast = prql_to_pl(source).map_err(|e| anyhow!(e))?;
-
- serde_yaml::to_string(&ast)?.into_bytes()
+ Command::Parse { format, .. } => {
+ let ast = prql_to_pl(source)?;
+ match format {
+ Some(Format::Json) | None => serde_json::to_string_pretty(&ast)?.into_bytes(),
+ Some(Format::Yaml) => serde_yaml::to_string(&ast)?.into_bytes(),
+ }
}
- Cli::Format(_) => prql_to_pl(source)
- .and_then(pl_to_prql)
- .map_err(|x| anyhow!(x))?
- .as_bytes()
- .to_vec(),
- Cli::Debug(_) => {
- let stmts = prql_to_pl(source).map_err(|x| anyhow!(x))?;
+ Command::Format(_) => prql_to_pl(source).and_then(pl_to_prql)?.as_bytes().to_vec(),
+ Command::Debug(_) => {
+ let stmts = prql_to_pl(source)?;
let (stmts, context) = semantic::resolve_only(stmts, None)?;
let (references, stmts) =
@@ -119,8 +143,11 @@ impl Cli {
]
.concat()
}
- Cli::Annotate(_) => {
- let stmts = prql_to_pl(source).map_err(|x| anyhow!(x))?;
+ Command::Annotate(_) => {
+ // TODO: potentially if there is code performing a role beyond
+ // presentation, it should be a library function; and we could
+ // promote it to the `prql-compiler` crate.
+ let stmts = prql_to_pl(source)?;
// resolve
let (stmts, _) = semantic::resolve_only(stmts, None)?;
@@ -130,49 +157,55 @@ impl Cli {
// combine with source
combine_prql_and_frames(source, frames).as_bytes().to_vec()
}
- Cli::Resolve(_) => {
- let ast = prql_to_pl(source).map_err(|x| anyhow!(x))?;
+ Command::Resolve(_) => {
+ // We can't currently have `--format=yaml` here, because
+ // serde_yaml is unable to serialize an Enum of an Enum; from
+ // https://github.com/dtolnay/serde-yaml/blob/68a9e95c9fd639498c85f55b5485f446b3f8465c/tests/test_error.rs#L175
+ let ast = prql_to_pl(source)?;
let ir = semantic::resolve(ast)?;
-
serde_json::to_string_pretty(&ir)?.into_bytes()
}
- Cli::Compile(_) => compile(source, Options::default())
- .map_or_else(|x| x.to_string(), |x| x)
- .as_bytes()
- .to_vec(),
- Cli::Watch(_) => unreachable!(),
+ // TODO: Allow passing the `Options` to the CLI; map those through.
+ // We already do this in Watch.
+ Command::Compile(_) => compile(source, &Options::default())?.as_bytes().to_vec(),
+ Command::Watch(_) => unreachable!(),
})
}
fn read_input(&mut self) -> Result<(String, String)> {
- use Cli::*;
- match self {
- Parse(io) | Format(io) | Debug(io) | Annotate(io) | Resolve(io) | Compile(io) => {
- // Don't wait without a prompt when running `prql-compiler compile` โ
- // it's confusing whether it's waiting for input or not. This
- // offers the prompt.
- if is_stdin(&io.input) && atty::is(atty::Stream::Stdin) {
- println!("Enter PRQL, then ctrl-d:");
- println!();
- }
-
- let mut source = String::new();
- io.input.read_to_string(&mut source)?;
- let source_id = (*io.input.path()).to_str().unwrap().to_string();
- Ok((source, source_id))
- }
- Cli::Watch(_) => unreachable!(),
+ // TODO: possibly this should be called by the relevant subcommands
+ // passing in `input`, rather than matching on them and grabbing `input`
+ // from `self`.
+ use Command::*;
+ let mut input = match self {
+ Parse { input, .. } => input.clone(),
+ Format(io) | Debug(io) | Annotate(io) | Resolve(io) | Compile(io) => io.input.clone(),
+ Watch(_) => unreachable!(),
+ };
+ // Don't wait without a prompt when running `prqlc compile` โ
+ // it's confusing whether it's waiting for input or not. This
+ // offers the prompt.
+ if is_stdin(&input) && atty::is(atty::Stream::Stdin) {
+ println!("Enter PRQL, then ctrl-d:");
+ println!();
}
+
+ let mut source = String::new();
+ (input).read_to_string(&mut source)?;
+ let source_id = (input.path()).to_str().unwrap().to_string();
+ Ok((source, source_id))
}
fn write_output(&mut self, data: &[u8]) -> std::io::Result<()> {
- use Cli::*;
- match self {
- Parse(io) | Format(io) | Debug(io) | Annotate(io) | Resolve(io) | Compile(io) => {
- io.output.write_all(data)
+ use Command::*;
+ let mut output = match self {
+ Parse { output, .. } => output.to_owned(),
+ Format(io) | Debug(io) | Annotate(io) | Resolve(io) | Compile(io) => {
+ io.output.to_owned()
}
- Cli::Watch(_) => unreachable!(),
- }
+ Watch(_) => unreachable!(),
+ };
+ output.write_all(data)
}
}
@@ -208,14 +241,30 @@ fn combine_prql_and_frames(source: &str, frames: Vec<(Span, Frame)>) -> String {
#[cfg(test)]
mod tests {
- use insta::assert_snapshot;
+ use insta::{assert_display_snapshot, assert_snapshot};
+
+ // TODO: would be good to test the basic CLI interface โ i.e. snapshotting this:
+
+ // $ prqlc parse --help
+ //
+ // Parse PL AST
+ //
+ // Usage: prqlc parse [OPTIONS] [INPUT] [OUTPUT]
+ //
+ // Arguments:
+ // [INPUT] [default: -]
+ // [OUTPUT] [default: -]
+ //
+ // Options:
+ // --format [possible values: json, yaml]
+ // -h, --help Print help
use super::*;
#[test]
fn layouts() {
- let output = Cli::execute(
- &Cli::Annotate(CommandIO::default()),
+ let output = Command::execute(
+ &Command::Annotate(IoArgs::default()),
r#"
from initial_table
select [f = first_name, l = last_name, gender]
@@ -239,8 +288,8 @@ sort full
#[test]
fn format() {
- let output = Cli::execute(
- &Cli::Format(CommandIO::default()),
+ let output = Command::execute(
+ &Command::Format(IoArgs::default()),
r#"
from table.subdivision
derive `ลพelva_means_turtle` = (`column with spaces` + 1) * 3
@@ -267,4 +316,53 @@ group a_column (take 10 | sort b_column | derive [the_number = rank, last = lag
)
"###);
}
+
+ #[test]
+ fn compile() {
+ // Check we get an error on a bad input
+ let input = "asdf";
+ let result = Command::execute(&Command::Compile(IoArgs::default()), input);
+ assert_display_snapshot!(result.unwrap_err(), @r###"
+ Error:
+ โญโ[:1:1]
+ โ
+ 1 โ asdf
+ ยท โโโฌโ
+ ยท โฐโโโ Unknown name asdf
+ โโโโฏ
+ "###);
+ }
+
+ #[test]
+ fn parse() {
+ let output = Command::execute(
+ &Command::Parse {
+ input: IoArgs::default().input,
+ output: IoArgs::default().output,
+ format: Some(Format::Yaml),
+ },
+ "from x | select y",
+ )
+ .unwrap();
+
+ assert_display_snapshot!(String::from_utf8(output).unwrap().trim(), @r###"
+ - Main:
+ Pipeline:
+ exprs:
+ - FuncCall:
+ name:
+ Ident:
+ - from
+ args:
+ - Ident:
+ - x
+ - FuncCall:
+ name:
+ Ident:
+ - select
+ args:
+ - Ident:
+ - y
+ "###);
+ }
}
diff --git a/prql-compiler/prqlc/src/jinja.rs b/prql-compiler/prqlc/src/jinja.rs
new file mode 100644
index 000000000000..6bec828644c8
--- /dev/null
+++ b/prql-compiler/prqlc/src/jinja.rs
@@ -0,0 +1,177 @@
+//! Handling of Jinja templates
+//!
+//! dbt is using the following pipeline: `Jinja+SQL -> SQL -> execution`.
+//!
+//! To prevent messing up the templates, we have create the following pipeline:
+//! ```
+//! Jinja+PRQL -> Jinja+SQL -> SQL -> execution
+//! ```
+//!
+//! But because prql-compiler does not (and should not) know how to handle Jinja,
+//! we have to extract the interpolations, replace them something that is valid PRQL,
+//! compile the query and inject interpolations back in.
+//!
+//! Unfortunately, this requires parsing Jinja.
+//!
+//! use crate::compiler::tokens::{Span, Token};
+
+use std::collections::HashMap;
+
+use anyhow::Result;
+use minijinja::machinery::{Span, Token};
+use regex::Regex;
+
+const ANCHOR_PREFIX: &str = "_jinja_";
+
+#[derive(Debug)]
+pub enum JinjaBlock<'a> {
+ Data(&'a str),
+ Interpolation(Vec<(Token<'a>, Span)>),
+}
+
+#[derive(Default)]
+pub struct JinjaContext<'a> {
+ anchor_map: HashMap,
+ header: Vec<&'a str>,
+}
+
+/// Parse source as Jinja template, extract all interpolations
+/// and replace them with anchors.
+pub fn pre_process(source: &str) -> Result<(String, JinjaContext)> {
+ let mut blocks = Vec::new();
+ let mut current_block = Vec::new();
+
+ for res in minijinja::machinery::tokenize(source, false) {
+ let (token, span) = res?;
+
+ if let Token::TemplateData(data) = token {
+ if !current_block.is_empty() {
+ blocks.push(JinjaBlock::Interpolation(current_block));
+ current_block = Vec::new();
+ }
+ blocks.push(JinjaBlock::Data(data))
+ } else {
+ current_block.push((token, span));
+ }
+ }
+ if !current_block.is_empty() {
+ blocks.push(JinjaBlock::Interpolation(current_block));
+ }
+
+ let mut anchored_source = String::new();
+ let mut next_anchor_id = 0;
+ let mut context = JinjaContext::default();
+ for block in blocks {
+ match block {
+ JinjaBlock::Data(data) => anchored_source += data,
+ JinjaBlock::Interpolation(block) => {
+ let (tokens, spans): (Vec<_>, _) = block.into_iter().unzip();
+
+ let source_span = find_span(source, spans);
+
+ if let Some(Token::Ident("config" | "set")) = tokens.get(1) {
+ context.header.push(source_span);
+ } else {
+ let id = format!("{ANCHOR_PREFIX}{next_anchor_id}");
+ next_anchor_id += 1;
+
+ anchored_source += &id;
+
+ context.anchor_map.insert(id, source_span);
+ }
+ }
+ }
+ }
+
+ Ok((anchored_source, context))
+}
+
+fn find_span(source: &str, spans: Vec) -> &str {
+ let start = spans.first().unwrap();
+ let end = spans.last().unwrap();
+
+ let mut start_index = 0;
+ let mut end_index = source.len();
+
+ let mut line = 1;
+ let mut col = 0;
+ for (index, char) in source.chars().enumerate() {
+ if char == '\n' {
+ line += 1;
+ col = 0;
+ continue;
+ } else {
+ col += 1;
+ }
+
+ if line == start.start_line && col == start.start_col {
+ start_index = index;
+ }
+ if line == end.end_line && col == end.end_col {
+ end_index = index + 1;
+ }
+ }
+ &source[start_index..end_index]
+}
+
+/// Replace anchors with their values.
+pub fn post_process(source: &str, context: JinjaContext) -> String {
+ let mut res = String::new();
+
+ for stmt in context.header {
+ res += stmt;
+ res += "\n";
+ }
+
+ let re = Regex::new(&format!(r"{ANCHOR_PREFIX}\d+")).unwrap();
+
+ let mut last_index = 0;
+ for cap in re.captures_iter(source) {
+ let cap = cap.get(0).unwrap();
+ let index = cap.start();
+ let anchor_id = cap.as_str();
+
+ res += &source[last_index..index];
+ res += context.anchor_map.get(anchor_id).unwrap_or(&anchor_id);
+
+ last_index = index + anchor_id.len();
+ }
+ res += &source[last_index..];
+
+ res
+}
+
+#[cfg(test)]
+mod test {
+ use super::Span;
+
+ #[test]
+ fn test_find_span() {
+ let text = r#"line 1 col 13
+ line 2 col 21
+ some text line 3 col 31 more text
+ "#;
+
+ assert_eq!(
+ super::find_span(
+ text,
+ vec![
+ Span {
+ start_line: 2,
+ start_col: 9,
+ end_line: 12123123,
+ end_col: 2930293,
+ },
+ Span {
+ start_line: 7893648,
+ start_col: 79678,
+ end_line: 3,
+ end_col: 31,
+ }
+ ]
+ ),
+ r#"line 2 col 21
+ some text line 3 col 31"#
+ );
+ }
+}
diff --git a/prql-compiler/prqlc/src/main.rs b/prql-compiler/prqlc/src/main.rs
index 98ed661a72d6..0a3e3d6edc14 100644
--- a/prql-compiler/prqlc/src/main.rs
+++ b/prql-compiler/prqlc/src/main.rs
@@ -7,6 +7,8 @@
#[cfg(not(target_family = "wasm"))]
mod cli;
#[cfg(not(target_family = "wasm"))]
+mod jinja;
+#[cfg(not(target_family = "wasm"))]
mod watch;
#[cfg(not(target_family = "wasm"))]
diff --git a/prql-compiler/prqlc/src/watch.rs b/prql-compiler/prqlc/src/watch.rs
index a6bd2c4d0a2b..a57322e116af 100644
--- a/prql-compiler/prqlc/src/watch.rs
+++ b/prql-compiler/prqlc/src/watch.rs
@@ -8,8 +8,10 @@ use notify::{Config, RecommendedWatcher, RecursiveMode, Watcher};
use prql_compiler::downcast;
use walkdir::WalkDir;
-#[derive(Parser)]
-pub struct WatchCommand {
+use crate::jinja;
+
+#[derive(Parser, Debug, Clone)]
+pub struct WatchArgs {
/// Directory or file to watch for changes
pub path: OsString,
@@ -20,7 +22,7 @@ pub struct WatchCommand {
pub no_signature: bool,
}
-pub fn run(command: &mut WatchCommand) -> Result<()> {
+pub fn run(command: &mut WatchArgs) -> Result<()> {
let opt = prql_compiler::Options {
format: !command.no_format,
target: prql_compiler::Target::Sql(None),
@@ -33,7 +35,7 @@ pub fn run(command: &mut WatchCommand) -> Result<()> {
// watch and compile
println!("Watching path \"{}\"", path.display());
- watch_and_compile(path, &opt).unwrap();
+ watch_and_compile(path, &opt)?;
Ok(())
}
@@ -108,9 +110,12 @@ fn compile_path(path: &Path, opt: &prql_compiler::Options) -> Result<()> {
return Ok(());
}
+ // pre-process Jinja
+ let (prql_string, jinja_context) = jinja::pre_process(&prql_string)?;
+
// compile
println!("Compiling {}", prql_path.display());
- let sql_string = match prql_compiler::compile(&prql_string, opt.clone()) {
+ let sql_string = match prql_compiler::compile(&prql_string, opt) {
Ok(sql_string) => sql_string,
Err(err) => {
let source_id = &prql_path.to_str().unwrap_or_default();
@@ -119,6 +124,9 @@ fn compile_path(path: &Path, opt: &prql_compiler::Options) -> Result<()> {
}
};
+ // post-process Jinja
+ let sql_string = jinja::post_process(&sql_string, jinja_context);
+
// write
fs::write(sql_path, sql_string)?;
diff --git a/prql-compiler/src/ast/pl/expr.rs b/prql-compiler/src/ast/pl/expr.rs
index ea3dc6f0a7b7..fc98216c51fa 100644
--- a/prql-compiler/src/ast/pl/expr.rs
+++ b/prql-compiler/src/ast/pl/expr.rs
@@ -72,11 +72,15 @@ pub enum ExprKind {
TransformCall(TransformCall),
SString(Vec),
FString(Vec),
- Switch(Vec),
+ Case(Vec),
BuiltInFunction {
name: String,
args: Vec,
},
+ Set(SetExpr),
+
+ /// a placeholder for values provided after query is compiled
+ Param(String),
}
impl ExprKind {
@@ -89,7 +93,16 @@ impl ExprKind {
}
#[derive(
- Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, strum::Display, strum::EnumString,
+ Debug,
+ PartialEq,
+ Eq,
+ Clone,
+ Copy,
+ Hash,
+ Serialize,
+ Deserialize,
+ strum::Display,
+ strum::EnumString,
)]
pub enum BinOp {
#[strum(to_string = "*")]
@@ -122,12 +135,12 @@ pub enum BinOp {
Coalesce,
}
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, strum::EnumString)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize, strum::EnumString)]
pub enum UnOp {
#[strum(to_string = "-")]
Neg,
#[strum(to_string = "+")]
- Add,
+ Add, // TODO: rename to Pos
#[strum(to_string = "!")]
Not,
#[strum(to_string = "==")]
@@ -142,6 +155,7 @@ pub struct ListItem(pub Expr);
pub struct FuncCall {
pub name: Box,
pub args: Vec,
+ #[serde(default, skip_serializing_if = "HashMap::is_empty")]
pub named_args: HashMap,
}
@@ -163,8 +177,8 @@ pub struct Closure {
pub body_ty: Option,
pub args: Vec,
- pub params: Vec,
- pub named_params: Vec,
+ pub params: Vec,
+ pub named_params: Vec,
pub env: HashMap,
}
@@ -177,6 +191,16 @@ impl Closure {
}
}
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub struct ClosureParam {
+ pub name: String,
+
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ty: Option,
+
+ pub default_value: Option,
+}
+
#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
pub struct WindowFrame> {
pub kind: WindowKind,
@@ -310,6 +334,7 @@ pub enum TransformKind {
pipeline: Box,
},
Append(Box),
+ Loop(Box),
}
#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
@@ -327,7 +352,7 @@ pub enum TableExternRef {
/// Placeholder for a relation that will be provided later.
/// This is very similar to relational s-strings and may not even be needed for now, so
/// it's not documented anywhere. But it will be used in the future.
- Anchor(String),
+ Param(String),
// TODO: add other sources such as files, URLs
}
@@ -430,7 +455,6 @@ impl From for anyhow::Error {
// https://github.com/bluejekyll/enum-as-inner/issues/84
#[allow(unreachable_code)]
fn from(kind: ExprKind) -> Self {
- // panic!("Failed to convert {item}")
anyhow!("Failed to convert `{}`", Expr::from(kind))
}
}
@@ -439,7 +463,6 @@ impl From for anyhow::Error {
// https://github.com/bluejekyll/enum-as-inner/issues/84
#[allow(unreachable_code)]
fn from(kind: TransformKind) -> Self {
- // panic!("Failed to convert {item}")
anyhow!("Failed to convert `{kind:?}`")
}
}
@@ -561,8 +584,8 @@ impl Display for Expr {
ExprKind::Literal(literal) => {
write!(f, "{}", literal)?;
}
- ExprKind::Switch(cases) => {
- f.write_str("switch [\n")?;
+ ExprKind::Case(cases) => {
+ f.write_str("case [\n")?;
for case in cases {
writeln!(f, " {} => {}", case.condition, case.value)?;
}
@@ -571,6 +594,12 @@ impl Display for Expr {
ExprKind::BuiltInFunction { .. } => {
f.write_str("")?;
}
+ ExprKind::Set(_) => {
+ writeln!(f, "")?;
+ }
+ ExprKind::Param(id) => {
+ writeln!(f, "${id}")?;
+ }
}
Ok(())
diff --git a/prql-compiler/src/ast/pl/fold.rs b/prql-compiler/src/ast/pl/fold.rs
index 3af403b8525d..75bb3c2cf27c 100644
--- a/prql-compiler/src/ast/pl/fold.rs
+++ b/prql-compiler/src/ast/pl/fold.rs
@@ -36,10 +36,16 @@ pub trait AstFold {
fn fold_exprs(&mut self, exprs: Vec) -> Result> {
exprs.into_iter().map(|node| self.fold_expr(node)).collect()
}
- fn fold_var_def(&mut self, table: VarDef) -> Result {
+ fn fold_var_def(&mut self, var_def: VarDef) -> Result {
Ok(VarDef {
- name: table.name,
- value: Box::new(self.fold_expr(*table.value)?),
+ name: var_def.name,
+ value: Box::new(self.fold_expr(*var_def.value)?),
+ })
+ }
+ fn fold_type_def(&mut self, ty_def: TypeDef) -> Result {
+ Ok(TypeDef {
+ name: ty_def.name,
+ value: ty_def.value.map(|x| self.fold_expr(x)).transpose()?,
})
}
fn fold_pipeline(&mut self, pipeline: Pipeline) -> Result {
@@ -61,7 +67,7 @@ pub trait AstFold {
fold_interpolate_item(self, sstring_item)
}
fn fold_type(&mut self, t: Ty) -> Result {
- fold_type(self, t)
+ Ok(t)
}
fn fold_window(&mut self, window: WindowFrame) -> Result {
fold_window(self, window)
@@ -100,7 +106,7 @@ pub fn fold_expr_kind(fold: &mut T, expr_kind: ExprKind) ->
.map(|x| fold.fold_interpolate_item(x))
.try_collect()?,
),
- Switch(cases) => Switch(fold_cases(fold, cases)?),
+ Case(cases) => Case(fold_cases(fold, cases)?),
FuncCall(func_call) => FuncCall(fold.fold_func_call(func_call)?),
Closure(closure) => Closure(Box::new(fold.fold_closure(*closure)?)),
@@ -110,9 +116,10 @@ pub fn fold_expr_kind(fold: &mut T, expr_kind: ExprKind) ->
name,
args: fold.fold_exprs(args)?,
},
+ Param(id) => Param(id),
// None of these capture variables, so we don't need to fold them.
- Literal(_) => expr_kind,
+ Literal(_) | Set(_) => expr_kind,
})
}
@@ -121,6 +128,7 @@ pub fn fold_stmt_kind(fold: &mut T, stmt_kind: StmtKind) ->
Ok(match stmt_kind {
FuncDef(func) => FuncDef(fold.fold_func_def(func)?),
VarDef(var_def) => VarDef(fold.fold_var_def(var_def)?),
+ TypeDef(type_def) => TypeDef(fold.fold_type_def(type_def)?),
Main(expr) => Main(Box::new(fold.fold_expr(*expr)?)),
QueryDef(_) => stmt_kind,
})
@@ -273,6 +281,7 @@ pub fn fold_transform_kind(
range: fold_range(fold, range)?,
pipeline: Box::new(fold.fold_expr(*pipeline)?),
},
+ Loop(pipeline) => Loop(Box::new(fold.fold_expr(*pipeline)?)),
})
}
@@ -312,14 +321,3 @@ pub fn fold_func_param(
})
.try_collect()
}
-
-pub fn fold_type(fold: &mut T, t: Ty) -> Result {
- Ok(match t {
- Ty::Literal(_) => t,
- Ty::Parameterized(t, p) => {
- Ty::Parameterized(Box::new(fold.fold_type(*t)?), Box::new(fold.fold_type(*p)?))
- }
- Ty::AnyOf(ts) => Ty::AnyOf(ts.into_iter().map(|t| fold_type(fold, t)).try_collect()?),
- _ => t,
- })
-}
diff --git a/prql-compiler/src/ast/pl/stmt.rs b/prql-compiler/src/ast/pl/stmt.rs
index 05c43066a00b..11f2c5124b77 100644
--- a/prql-compiler/src/ast/pl/stmt.rs
+++ b/prql-compiler/src/ast/pl/stmt.rs
@@ -27,6 +27,7 @@ pub enum StmtKind {
QueryDef(QueryDef),
FuncDef(FuncDef),
VarDef(VarDef),
+ TypeDef(TypeDef),
Main(Box),
}
@@ -44,15 +45,16 @@ pub struct FuncDef {
pub positional_params: Vec, // ident
pub named_params: Vec, // named expr
pub body: Box,
- pub return_ty: Option,
+ pub return_ty: Option,
}
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
pub struct FuncParam {
pub name: String,
+ /// Parsed expression that will be resolved to a type
#[serde(skip_serializing_if = "Option::is_none")]
- pub ty: Option,
+ pub ty_expr: Option,
pub default_value: Option,
}
@@ -63,6 +65,12 @@ pub struct VarDef {
pub value: Box,
}
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub struct TypeDef {
+ pub name: String,
+ pub value: Option,
+}
+
impl From for Stmt {
fn from(kind: StmtKind) -> Self {
Stmt {
@@ -127,6 +135,13 @@ impl Display for StmtKind {
}
};
}
+ StmtKind::TypeDef(ty_def) => {
+ if let Some(value) = &ty_def.value {
+ write!(f, "type {} = {value}\n\n", ty_def.name)?;
+ } else {
+ write!(f, "type {}\n\n", ty_def.name)?;
+ }
+ }
}
Ok(())
}
diff --git a/prql-compiler/src/ast/pl/types.rs b/prql-compiler/src/ast/pl/types.rs
index 3d4c428d5d75..622b3117020a 100644
--- a/prql-compiler/src/ast/pl/types.rs
+++ b/prql-compiler/src/ast/pl/types.rs
@@ -1,20 +1,49 @@
-use std::cmp::Ordering;
use std::fmt::{Debug, Display, Formatter, Result, Write};
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Serialize};
-use super::Frame;
+use super::{Frame, Literal};
-#[derive(Clone, PartialEq, Serialize, Deserialize, EnumAsInner)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, EnumAsInner)]
+pub enum SetExpr {
+ /// Set of a built-in primitive type
+ Primitive(TyLit),
+
+ /// Set that contains only a literal value
+ Singleton(Literal),
+
+ /// Union of sets (sum)
+ Union(Vec<(Option, SetExpr)>),
+
+ /// Set of tuples (product)
+ Tuple(Vec),
+
+ /// Set of arrays
+ Array(Box),
+
+ /// Set of sets.
+ /// Used for exprs that can be converted to SetExpr and then used as a Ty.
+ Set,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum TupleElement {
+ Single(Option, SetExpr),
+ Wildcard,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, EnumAsInner)]
pub enum Ty {
- Empty,
- Literal(TyLit),
- Named(String),
- Parameterized(Box, Box),
- AnyOf(Vec),
+ /// Value is an element of this [SetExpr]
+ SetExpr(SetExpr),
+
+ /// Value is a function described by [TyFunc]
+ // TODO: convert into [Ty::Domain].
Function(TyFunc),
+ /// Special type for relations.
+ // TODO: convert into [Ty::Domain].
Table(Frame),
/// Means that we have no information about the type of the variable and
@@ -26,20 +55,23 @@ pub enum Ty {
Debug, Clone, Serialize, Deserialize, PartialEq, Eq, strum::EnumString, strum::Display,
)]
pub enum TyLit {
+ // TODO: convert to a named expression
#[strum(to_string = "list")]
List,
+ // TODO: convert to a named expression
#[strum(to_string = "column")]
Column,
+ // TODO: convert to a named expression
#[strum(to_string = "scalar")]
Scalar,
- #[strum(to_string = "integer")]
- Integer,
+ #[strum(to_string = "int")]
+ Int,
#[strum(to_string = "float")]
Float,
#[strum(to_string = "bool")]
Bool,
- #[strum(to_string = "string")]
- String,
+ #[strum(to_string = "text")]
+ Text,
#[strum(to_string = "date")]
Date,
#[strum(to_string = "time")]
@@ -56,76 +88,33 @@ pub struct TyFunc {
}
impl Ty {
- pub const fn column() -> Ty {
- Ty::Literal(TyLit::Column)
- }
-}
+ pub fn is_superset_of(&self, subset: &Ty) -> bool {
+ match (self, subset) {
+ // Not handled here. See type_resolver.
+ (Ty::Infer, _) | (_, Ty::Infer) => false,
-impl From for Ty {
- fn from(lit: TyLit) -> Self {
- Ty::Literal(lit)
- }
-}
+ (Ty::SetExpr(left), Ty::SetExpr(right)) => left.is_superset_of(right),
-impl Default for Ty {
- fn default() -> Self {
- Ty::Infer
+ (Ty::Table(_), Ty::Table(_)) => true,
+
+ (l, r) => l == r,
+ }
}
}
-/// Implements a partial ordering or types:
-/// - higher up are types that include many others (AnyOf, Any) and
-/// - on the bottom are the atomic types (bool, string).
-impl PartialOrd for Ty {
- fn partial_cmp(&self, other: &Self) -> Option {
- match (self, other) {
- // Not handled here. See type_resolver.
- (Ty::Infer, _) | (_, Ty::Infer) => None,
-
- (Ty::Literal(TyLit::Column), Ty::Literal(TyLit::Column)) => Some(Ordering::Equal),
- (Ty::Literal(TyLit::Column), Ty::Literal(_)) => Some(Ordering::Greater),
- (Ty::Literal(_), Ty::Literal(TyLit::Column)) => Some(Ordering::Less),
-
- (Ty::Literal(l0), Ty::Literal(r0)) => {
- if l0 == r0 {
- Some(Ordering::Equal)
- } else {
- None
- }
- }
- (Ty::AnyOf(many), one) => {
- if many.iter().any(|m| m >= one) {
- Some(Ordering::Greater)
- } else {
- None
- }
- }
- (one, Ty::AnyOf(many)) => {
- if many.iter().any(|m| m >= one) {
- Some(Ordering::Less)
- } else {
- None
- }
- }
- (Ty::Parameterized(l_ty, l_param), Ty::Parameterized(r_ty, r_param)) => {
- if l_ty == r_ty && l_param == r_param {
- Some(Ordering::Equal)
- } else {
- None
- }
- }
- (Ty::Parameterized(l_ty, _), r_ty) if **l_ty == *r_ty => Some(Ordering::Equal),
- (l_ty, Ty::Parameterized(r_ty, _)) if *l_ty == **r_ty => Some(Ordering::Equal),
+impl SetExpr {
+ fn is_superset_of(&self, subset: &SetExpr) -> bool {
+ match (self, subset) {
+ // TODO: convert these to array
+ (SetExpr::Primitive(TyLit::Column), SetExpr::Primitive(TyLit::Column)) => true,
+ (SetExpr::Primitive(TyLit::Column), SetExpr::Primitive(_)) => true,
+ (SetExpr::Primitive(_), SetExpr::Primitive(TyLit::Column)) => false,
- (Ty::Table(_), Ty::Table(_)) => Some(Ordering::Equal),
+ (SetExpr::Primitive(l0), SetExpr::Primitive(r0)) => l0 == r0,
+ (SetExpr::Union(many), one) => many.iter().any(|(_, any)| any.is_superset_of(one)),
+ (one, SetExpr::Union(many)) => many.iter().all(|(_, each)| one.is_superset_of(each)),
- (l, r) => {
- if l == r {
- Some(Ordering::Equal)
- } else {
- None
- }
- }
+ (l, r) => l == r,
}
}
}
@@ -133,21 +122,7 @@ impl PartialOrd for Ty {
impl Display for Ty {
fn fmt(&self, f: &mut Formatter<'_>) -> Result {
match &self {
- Ty::Empty => write!(f, "()"),
- Ty::Literal(lit) => write!(f, "{:}", lit),
- Ty::Named(name) => write!(f, "{:}", name),
- Ty::Parameterized(t, param) => {
- write!(f, "{t}<{}>", param)
- }
- Ty::AnyOf(ts) => {
- for (i, t) in ts.iter().enumerate() {
- write!(f, "{t}")?;
- if i < ts.len() - 1 {
- f.write_char('|')?;
- }
- }
- Ok(())
- }
+ Ty::SetExpr(lit) => write!(f, "{:}", lit),
Ty::Table(frame) => write!(f, "table<{frame}>"),
Ty::Infer => write!(f, "infer"),
Ty::Function(func) => {
@@ -163,8 +138,40 @@ impl Display for Ty {
}
}
-impl Debug for Ty {
+impl Display for SetExpr {
fn fmt(&self, f: &mut Formatter<'_>) -> Result {
- Display::fmt(self, f)
+ match &self {
+ SetExpr::Primitive(lit) => write!(f, "{:}", lit),
+ SetExpr::Union(ts) => {
+ for (i, (_, e)) in ts.iter().enumerate() {
+ write!(f, "{e}")?;
+ if i < ts.len() - 1 {
+ f.write_char('|')?;
+ }
+ }
+ Ok(())
+ }
+ SetExpr::Singleton(lit) => write!(f, "{:}", lit),
+ SetExpr::Tuple(elements) => {
+ write!(f, "[")?;
+ for e in elements {
+ match e {
+ TupleElement::Wildcard => {
+ write!(f, "*")?;
+ }
+ TupleElement::Single(name, expr) => {
+ if let Some(name) = name {
+ write!(f, "{name} = ")?
+ }
+ write!(f, "{expr}")?
+ }
+ }
+ write!(f, ",")?
+ }
+ Ok(())
+ }
+ SetExpr::Set => write!(f, "set"),
+ SetExpr::Array(_) => todo!(),
+ }
}
}
diff --git a/prql-compiler/src/ast/rq/expr.rs b/prql-compiler/src/ast/rq/expr.rs
index bf70f4745ffd..d685459f268c 100644
--- a/prql-compiler/src/ast/rq/expr.rs
+++ b/prql-compiler/src/ast/rq/expr.rs
@@ -34,12 +34,15 @@ pub enum ExprKind {
// TODO: convert this into built-in function
FString(Vec>),
- Switch(Vec>),
+ Case(Vec>),
BuiltInFunction {
name: String,
args: Vec,
},
+
+ /// Placeholder for
+ Param(String),
}
#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
diff --git a/prql-compiler/src/ast/rq/fold.rs b/prql-compiler/src/ast/rq/fold.rs
index d84d6385ba37..d0a35d89260a 100644
--- a/prql-compiler/src/ast/rq/fold.rs
+++ b/prql-compiler/src/ast/rq/fold.rs
@@ -187,6 +187,7 @@ pub fn fold_transform(
filter: fold.fold_expr(filter)?,
},
Append(bottom) => Append(fold.fold_table_ref(bottom)?),
+ Loop(transforms) => Loop(fold_transforms(fold, transforms)?),
};
Ok(transform)
}
@@ -222,7 +223,7 @@ pub fn fold_expr_kind(fold: &mut F, kind: ExprKind) -> Resul
ExprKind::SString(items) => ExprKind::SString(fold_interpolate_items(fold, items)?),
ExprKind::FString(items) => ExprKind::FString(fold_interpolate_items(fold, items)?),
- ExprKind::Switch(cases) => ExprKind::Switch(
+ ExprKind::Case(cases) => ExprKind::Case(
cases
.into_iter()
.map(|c| fold_switch_case(fold, c))
@@ -232,6 +233,7 @@ pub fn fold_expr_kind(fold: &mut F, kind: ExprKind) -> Resul
name,
args: args.into_iter().map(|a| fold.fold_expr(a)).try_collect()?,
},
+ ExprKind::Param(id) => ExprKind::Param(id),
ExprKind::Literal(_) => kind,
})
diff --git a/prql-compiler/src/ast/rq/ids.rs b/prql-compiler/src/ast/rq/ids.rs
index ecd27fb7245d..faced8329f5d 100644
--- a/prql-compiler/src/ast/rq/ids.rs
+++ b/prql-compiler/src/ast/rq/ids.rs
@@ -1,7 +1,7 @@
use serde::{Deserialize, Serialize};
/// Column id
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Ord, PartialOrd)]
pub struct CId(usize);
impl CId {
diff --git a/prql-compiler/src/ast/rq/transform.rs b/prql-compiler/src/ast/rq/transform.rs
index 7a946e1d2c13..875bc3d922b6 100644
--- a/prql-compiler/src/ast/rq/transform.rs
+++ b/prql-compiler/src/ast/rq/transform.rs
@@ -25,6 +25,7 @@ pub enum Transform {
filter: Expr,
},
Append(TableRef),
+ Loop(Vec),
}
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
diff --git a/prql-compiler/src/error.rs b/prql-compiler/src/error.rs
index e9f9d9d78839..aa656e45fe71 100644
--- a/prql-compiler/src/error.rs
+++ b/prql-compiler/src/error.rs
@@ -6,9 +6,6 @@ use std::error::Error as StdError;
use std::fmt::{self, Debug, Display, Formatter};
use std::ops::{Add, Range};
-use crate::parser::PestError;
-use crate::utils::IntoOnly;
-
#[derive(Clone, PartialEq, Eq, Copy, Serialize, Deserialize)]
pub struct Span {
pub start: usize,
@@ -20,8 +17,12 @@ pub struct Error {
pub span: Option,
pub reason: Reason,
pub help: Option,
+ pub code: Option<&'static str>,
}
+#[derive(Debug, Clone)]
+pub struct Errors(pub Vec);
+
/// Location within the source file.
/// Tuples contain:
/// - line number (0-based),
@@ -56,6 +57,7 @@ impl Error {
span: None,
reason,
help: None,
+ code: None,
}
}
@@ -72,10 +74,17 @@ impl Error {
self.span = span;
self
}
+
+ pub fn with_code(mut self, code: &'static str) -> Self {
+ self.code = Some(code);
+ self
+ }
}
-#[derive(Debug, Clone, Serialize)]
+#[derive(Clone, Serialize)]
pub struct ErrorMessage {
+ /// Machine-readable identifier of the error
+ pub code: Option,
/// Plain text of the error
pub reason: String,
/// A list of suggestions of how to fix the error
@@ -100,15 +109,28 @@ impl Display for ErrorMessage {
.join("\n");
f.write_str(&message_without_trailing_spaces)?;
} else {
- f.write_str(&self.reason)?;
+ let code = (self.code.as_ref())
+ .map(|c| format!("[{c}] "))
+ .unwrap_or_default();
+
+ writeln!(f, "{}Error: {}", code, &self.reason)?;
}
Ok(())
}
}
+impl Debug for ErrorMessage {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ Display::fmt(&self, f)
+ }
+}
+
// Needed for anyhow
impl StdError for Error {}
+// Needed for anyhow
+impl StdError for Errors {}
+
// Needed for StdError
impl Display for Error {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
@@ -116,10 +138,18 @@ impl Display for Error {
}
}
+// Needed for StdError
+impl Display for Errors {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ Debug::fmt(&self, f)
+ }
+}
+
#[derive(Debug, Clone, Serialize)]
pub struct ErrorMessages {
pub inner: Vec,
}
+impl StdError for ErrorMessages {}
impl From for ErrorMessages {
fn from(e: ErrorMessage) -> Self {
@@ -137,6 +167,7 @@ impl Display for ErrorMessages {
}
pub fn downcast(error: anyhow::Error) -> ErrorMessages {
+ let mut code = None;
let mut span = None;
let mut hint = None;
@@ -145,33 +176,35 @@ pub fn downcast(error: anyhow::Error) -> ErrorMessages {
Err(error) => error,
};
+ let error = match error.downcast::() {
+ Ok(messages) => {
+ return ErrorMessages {
+ inner: messages
+ .0
+ .into_iter()
+ .flat_map(|e| downcast(e.into()).inner)
+ .collect(),
+ }
+ }
+ Err(error) => error,
+ };
+
let reason = match error.downcast::() {
Ok(error) => {
+ code = error.code.map(|x| x.to_string());
span = error.span;
hint = error.help;
error.reason.message()
}
Err(error) => {
- match error.downcast::() {
- Ok(error) => {
- let range = pest::as_range(&error);
- span = Some(Span {
- start: range.start,
- end: range.end,
- });
-
- pest::as_message(&error)
- }
- Err(error) => {
- // default to basic Display
- format!("{:#?}", error)
- }
- }
+ // default to basic Display
+ format!("{:#?}", error)
}
};
ErrorMessage {
+ code,
reason,
hint,
span,
@@ -181,6 +214,12 @@ pub fn downcast(error: anyhow::Error) -> ErrorMessages {
.into()
}
+impl From for ErrorMessages {
+ fn from(e: anyhow::Error) -> Self {
+ downcast(e)
+ }
+}
+
impl ErrorMessages {
pub fn to_json(&self) -> String {
serde_json::to_string(self).unwrap()
@@ -199,14 +238,6 @@ impl ErrorMessages {
}
}
-impl IntoOnly for ErrorMessages {
- type Item = ErrorMessage;
-
- fn into_only(self) -> Result {
- self.inner.into_only()
- }
-}
-
impl ErrorMessage {
fn compose_display<'a, C>(&self, source_id: &'a str, cache: C, color: bool) -> Option
where
@@ -218,9 +249,12 @@ impl ErrorMessage {
let mut report = Report::build(ReportKind::Error, source_id, span.start)
.with_config(config)
- .with_message("")
.with_label(Label::new((source_id, span)).with_message(&self.reason));
+ if let Some(code) = &self.code {
+ report = report.with_code(code);
+ }
+
if let Some(hint) = &self.hint {
report.set_help(hint);
}
@@ -260,59 +294,6 @@ impl Reason {
}
}
-mod pest {
- use pest::error::{ErrorVariant, InputLocation};
- use std::ops::Range;
-
- use crate::parser::{PestError, PestRule};
-
- pub fn as_range(error: &PestError) -> Range {
- match error.location {
- InputLocation::Pos(r) => r..r + 1,
- InputLocation::Span(r) => r.0..r.1,
- }
- }
-
- pub fn as_message(error: &PestError) -> String {
- match error.variant {
- ErrorVariant::ParsingError {
- ref positives,
- ref negatives,
- } => parsing_error_message(positives, negatives),
- ErrorVariant::CustomError { ref message } => message.clone(),
- }
- }
-
- fn parsing_error_message(positives: &[PestRule], negatives: &[PestRule]) -> String {
- match (negatives.is_empty(), positives.is_empty()) {
- (false, false) => format!(
- "unexpected {}; expected {}",
- enumerate(negatives),
- enumerate(positives)
- ),
- (false, true) => format!("unexpected {}", enumerate(negatives)),
- (true, false) => format!("expected {}", enumerate(positives)),
- (true, true) => "unknown parsing error".to_owned(),
- }
- }
-
- fn enumerate(rules: &[PestRule]) -> String {
- match rules.len() {
- 1 => format!("{:?}", rules[0]),
- 2 => format!("{:?} or {:?}", rules[0], rules[1]),
- l => {
- let separated = rules
- .iter()
- .take(l - 1)
- .map(|x| format!("{:?}", x))
- .collect::>()
- .join(", ");
- format!("{}, or {:?}", separated, rules[l - 1])
- }
- }
- }
-}
-
impl From for Range {
fn from(a: Span) -> Self {
a.start..a.end
diff --git a/prql-compiler/src/lib.rs b/prql-compiler/src/lib.rs
index 2288b537b201..f88d4f133281 100644
--- a/prql-compiler/src/lib.rs
+++ b/prql-compiler/src/lib.rs
@@ -42,7 +42,7 @@
//! # fn main() -> Result<(), prql_compiler::ErrorMessages> {
//! let sql = prql_compiler::compile(
//! "from albums | select [title, artist_id]",
-//! prql_compiler::Options::default().no_format()
+//! &prql_compiler::Options::default().no_format()
//! )?;
//! assert_eq!(&sql[..35], "SELECT title, artist_id FROM albums");
//! # Ok(())
@@ -87,7 +87,6 @@ mod test;
mod utils;
pub use error::{downcast, Error, ErrorMessage, ErrorMessages, Reason, SourceLocation, Span};
-pub use utils::IntoOnly;
use once_cell::sync::Lazy;
use semver::Version;
@@ -109,19 +108,19 @@ pub static PRQL_VERSION: Lazy =
/// ```
/// use prql_compiler::{compile, Options, Target, sql::Dialect};
///
-/// let prql = "from employees | select [name,age] ";
-/// let opt = Options {
+/// let prql = "from employees | select [name,age]";
+/// let opts = Options {
/// format: false,
/// target: Target::Sql(Some(Dialect::SQLite)),
/// signature_comment: false
/// };
-/// let sql = compile(&prql, opt).unwrap();
+/// let sql = compile(&prql, &opts).unwrap();
/// println!("PRQL: {}\nSQLite: {}", prql, &sql);
/// assert_eq!("SELECT name, age FROM employees", sql)
///
/// ```
/// See [`sql::Options`](sql/struct.Options.html) and [`sql::Dialect`](sql/enum.Dialect.html) for options and supported SQL dialects.
-pub fn compile(prql: &str, options: Options) -> Result {
+pub fn compile(prql: &str, options: &Options) -> Result {
parser::parse(prql)
.and_then(semantic::resolve)
.and_then(|rq| sql::compile(rq, options))
@@ -143,10 +142,12 @@ impl Default for Target {
impl Target {
pub fn names() -> Vec {
- sql::Dialect::names()
- .into_iter()
- .map(|d| format!("sql.{d}"))
- .collect()
+ let mut names = vec!["sql.any".to_string()];
+
+ let dialects = sql::Dialect::names();
+ names.extend(dialects.into_iter().map(|d| format!("sql.{d}")));
+
+ names
}
}
@@ -154,20 +155,20 @@ impl FromStr for Target {
type Err = Error;
fn from_str(s: &str) -> Result {
- // We have a closure here because we can't create the error in the
- // pipeline, since it needs to be in two places, and we'd need to clone.
- // (Though possibly it's too optimize-y.)
- let not_found_error = |s| {
- Error::new(Reason::NotFound {
- name: format!("{s:?}"),
- namespace: "target".to_string(),
- })
- };
- s.strip_prefix("sql.")
- .ok_or_else(|| not_found_error(s))
- .map(sql::Dialect::from_str)?
- .map(|x| Target::Sql(Some(x)))
- .map_err(|_| not_found_error(s))
+ if let Some(dialect) = s.strip_prefix("sql.") {
+ if dialect == "any" {
+ return Ok(Target::Sql(None));
+ }
+
+ if let Ok(dialect) = sql::Dialect::from_str(dialect) {
+ return Ok(Target::Sql(Some(dialect)));
+ }
+ }
+
+ Err(Error::new(Reason::NotFound {
+ name: format!("{s:?}"),
+ namespace: "target".to_string(),
+ }))
}
}
@@ -233,12 +234,12 @@ pub fn prql_to_pl(prql: &str) -> Result, ErrorMessages> {
/// Perform semantic analysis and convert PL to RQ.
pub fn pl_to_rq(pl: Vec) -> Result {
- semantic::resolve(pl).map_err(error::downcast)
+ semantic::resolve(pl).map_err(|e| e.into())
}
/// Generate SQL from RQ.
-pub fn rq_to_sql(rq: ast::rq::Query, options: Options) -> Result {
- sql::compile(rq, options).map_err(error::downcast)
+pub fn rq_to_sql(rq: ast::rq::Query, options: &Options) -> Result {
+ sql::compile(rq, options).map_err(|e| e.into())
}
/// Generate PRQL code from PL AST
@@ -252,22 +253,22 @@ pub mod json {
/// JSON serialization
pub fn from_pl(pl: Vec) -> Result {
- serde_json::to_string(&pl).map_err(|e| error::downcast(anyhow::anyhow!(e)))
+ serde_json::to_string(&pl).map_err(|e| anyhow::anyhow!(e).into())
}
/// JSON deserialization
pub fn to_pl(json: &str) -> Result, ErrorMessages> {
- serde_json::from_str(json).map_err(|e| error::downcast(anyhow::anyhow!(e)))
+ serde_json::from_str(json).map_err(|e| anyhow::anyhow!(e).into())
}
/// JSON serialization
pub fn from_rq(rq: ast::rq::Query) -> Result {
- serde_json::to_string(&rq).map_err(|e| error::downcast(anyhow::anyhow!(e)))
+ serde_json::to_string(&rq).map_err(|e| anyhow::anyhow!(e).into())
}
/// JSON deserialization
pub fn to_rq(json: &str) -> Result {
- serde_json::from_str(json).map_err(|e| error::downcast(anyhow::anyhow!(e)))
+ serde_json::from_str(json).map_err(|e| anyhow::anyhow!(e).into())
}
}
@@ -298,6 +299,7 @@ mod tests {
namespace: "target",
},
help: None,
+ code: None,
},
)
"###);
@@ -311,6 +313,7 @@ mod tests {
namespace: "target",
},
help: None,
+ code: None,
},
)
"###);
diff --git a/prql-compiler/src/parser/chumsky.rs b/prql-compiler/src/parser/chumsky.rs
deleted file mode 100644
index 95c031453c5e..000000000000
--- a/prql-compiler/src/parser/chumsky.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-#![allow(dead_code)]
-
-use chumsky::prelude::*;
-
-use crate::ast::pl::*;
-
-fn str(chars: &str) -> impl Parser> + '_ {
- just(chars).to(())
-}
-
-fn operator() -> impl Parser> {
- operator_binary().to(()).or(operator_unary().to(()))
-}
-
-fn operator_binary() -> impl Parser> {
- operator_mul()
- .or(operator_add())
- .or(operator_compare())
- .or(operator_logical())
- .or(operator_coalesce())
-}
-fn operator_unary() -> impl Parser> {
- just('-')
- .to(UnOp::Neg)
- .or(just('+').to(UnOp::Add))
- .or(just('!').to(UnOp::Not))
- .or(str("==").to(UnOp::EqSelf))
-}
-fn operator_mul() -> impl Parser> {
- (just('*').to(BinOp::Mul))
- .or(just('/').to(BinOp::Div))
- .or(just('%').to(BinOp::Mod))
-}
-fn operator_add() -> impl Parser> {
- just('+').to(BinOp::Add).or(just('-').to(BinOp::Sub))
-}
-fn operator_compare() -> impl Parser> {
- str("==")
- .to(BinOp::Eq)
- .or(str("!=").to(BinOp::Ne))
- .or(str(">=").to(BinOp::Gte))
- .or(str("<=").to(BinOp::Lte))
- .or(str(">").to(BinOp::Gt))
- .or(str("<").to(BinOp::Lt))
-}
-fn operator_logical() -> impl Parser> {
- (just("and").to(BinOp::Add))
- .or(just("or").to(BinOp::Or))
- .then_ignore(text::whitespace())
-}
-fn operator_coalesce() -> impl Parser> {
- just("??").map(|_| BinOp::Coalesce)
-}
-
-fn ident_part() -> impl Parser> {
- let plain = filter(|c: &char| c.is_ascii_alphabetic() || *c == '_' || *c == '$')
- .map(Some)
- .chain::, _>(
- filter(|c: &char| c.is_ascii_alphanumeric() || *c == '_').repeated(),
- )
- .collect();
-
- let backticks = just('`')
- .ignore_then(filter(|c| *c != '`').repeated())
- .then_ignore(just('`'))
- .collect::();
-
- plain.or(backticks)
-}
-
-pub fn ident() -> impl Parser> {
- let star = just('*').map(|c| c.to_string());
-
- // TODO: !operator ~ !(keyword ~ WHITESPACE)
- // we probably need combinator::Not, which has not been released yet.
-
- ident_part()
- .chain(just('.').ignore_then(ident_part().or(star)).repeated())
- .map(Ident::from_path::)
-}
diff --git a/prql-compiler/src/parser/expr.rs b/prql-compiler/src/parser/expr.rs
new file mode 100644
index 000000000000..27840e0d26a4
--- /dev/null
+++ b/prql-compiler/src/parser/expr.rs
@@ -0,0 +1,315 @@
+use std::collections::HashMap;
+
+use chumsky::prelude::*;
+
+use crate::ast::pl::*;
+
+use super::common::*;
+use super::interpolation;
+use super::lexer::Token;
+
+pub fn expr_call() -> impl Parser> {
+ func_call(expr())
+}
+
+pub fn expr() -> impl Parser> + Clone {
+ recursive(|expr| {
+ let literal = select! { Token::Literal(lit) => ExprKind::Literal(lit) };
+
+ let ident_kind = ident().map(ExprKind::Ident);
+
+ let nested_expr = pipeline(func_call(expr.clone())).boxed();
+
+ let list = ident_part()
+ .then_ignore(ctrl('='))
+ .or_not()
+ .then(nested_expr.clone().map_with_span(into_expr))
+ .map(|(alias, expr)| Expr { alias, ..expr })
+ .padded_by(new_line().repeated())
+ .separated_by(ctrl(','))
+ .allow_trailing()
+ .then_ignore(new_line().repeated())
+ .delimited_by(ctrl('['), ctrl(']'))
+ .recover_with(nested_delimiters(
+ Token::Control('['),
+ Token::Control(']'),
+ [
+ (Token::Control('['), Token::Control(']')),
+ (Token::Control('('), Token::Control(')')),
+ ],
+ |_| vec![],
+ ))
+ .map(ExprKind::List)
+ .labelled("list");
+
+ let pipeline =
+ nested_expr
+ .delimited_by(ctrl('('), ctrl(')'))
+ .recover_with(nested_delimiters(
+ Token::Control('('),
+ Token::Control(')'),
+ [
+ (Token::Control('['), Token::Control(']')),
+ (Token::Control('('), Token::Control(')')),
+ ],
+ |_| Expr::null().kind,
+ ));
+
+ let interpolation =
+ select! {
+ Token::Interpolation('s', string) => (ExprKind::SString as fn(_) -> _, string),
+ Token::Interpolation('f', string) => (ExprKind::FString as fn(_) -> _, string),
+ }
+ .validate(|(finish, string), span: std::ops::Range, emit| {
+ match interpolation::parse(string, span.start + 2) {
+ Ok(items) => finish(items),
+ Err(errors) => {
+ for err in errors {
+ emit(err)
+ }
+ finish(vec![])
+ }
+ }
+ });
+
+ let case = keyword("case")
+ .ignore_then(
+ func_call(expr.clone())
+ .then_ignore(just(Token::ArrowDouble))
+ .then(func_call(expr))
+ .map(|(condition, value)| SwitchCase { condition, value })
+ .padded_by(new_line().repeated())
+ .separated_by(ctrl(','))
+ .allow_trailing()
+ .then_ignore(new_line().repeated())
+ .delimited_by(ctrl('['), ctrl(']')),
+ )
+ .map(ExprKind::Case);
+
+ let param = select! { Token::Param(id) => ExprKind::Param(id) };
+
+ let term = choice((
+ literal,
+ list,
+ pipeline,
+ interpolation,
+ ident_kind,
+ case,
+ param,
+ ))
+ .map_with_span(into_expr)
+ .boxed();
+
+ // Unary operators
+ let term = term
+ .clone()
+ .or(operator_unary()
+ .then(term.map(Box::new))
+ .map(|(op, expr)| ExprKind::Unary { op, expr })
+ .map_with_span(into_expr))
+ .boxed();
+
+ // Ranges have five cases we need to parse:
+ // x..y (bounded)
+ // x.. (only start bound)
+ // x (no-op)
+ // ..y (only end bound)
+ // .. (unbounded)
+ #[derive(Clone)]
+ enum RangeCase {
+ NoOp(Expr),
+ Range(Option, Option),
+ }
+ let term = choice((
+ // with start bound (first 3 cases)
+ term.clone()
+ .then(choice((
+ // range and end bound
+ just(Token::range(true, true))
+ .ignore_then(term.clone())
+ .map(|x| Some(Some(x))),
+ // range and no end bound
+ select! { Token::Range { bind_left: true, .. } => Some(None) },
+ // no range
+ empty().to(None),
+ )))
+ .map(|(start, range)| {
+ if let Some(end) = range {
+ RangeCase::Range(Some(start), end)
+ } else {
+ RangeCase::NoOp(start)
+ }
+ }),
+ // only end bound
+ select! { Token::Range { bind_right: true, .. } => () }
+ .ignore_then(term)
+ .map(|range| RangeCase::Range(None, Some(range))),
+ // unbounded
+ select! { Token::Range { .. } => RangeCase::Range(None, None) },
+ ))
+ .map_with_span(|case, span| match case {
+ RangeCase::NoOp(x) => x,
+ RangeCase::Range(start, end) => {
+ let kind = ExprKind::Range(Range {
+ start: start.map(Box::new),
+ end: end.map(Box::new),
+ });
+ into_expr(kind, span)
+ }
+ })
+ .boxed();
+
+ // Binary operators
+ let expr = term;
+ let expr = binary_op_parser(expr, operator_mul());
+ let expr = binary_op_parser(expr, operator_add());
+ let expr = binary_op_parser(expr, operator_compare());
+ let expr = binary_op_parser(expr, operator_coalesce());
+ let expr = binary_op_parser(expr, operator_and());
+
+ binary_op_parser(expr, operator_or())
+ })
+}
+
+pub fn pipeline(expr: E) -> impl Parser>
+where
+ E: Parser>,
+{
+ // expr has to be a param, because it can be either a normal expr() or
+ // a recursive expr called from within expr()
+
+ new_line()
+ .repeated()
+ .ignore_then(
+ expr.separated_by(ctrl('|').or(new_line().repeated().at_least(1).ignored()))
+ .at_least(1)
+ .map(|mut exprs| {
+ if exprs.len() == 1 {
+ exprs.remove(0).kind
+ } else {
+ ExprKind::Pipeline(Pipeline { exprs })
+ }
+ }),
+ )
+ .then_ignore(new_line().repeated())
+ .labelled("pipeline")
+}
+
+pub fn binary_op_parser<'a, Term, Op>(
+ term: Term,
+ op: Op,
+) -> impl Parser> + 'a
+where
+ Term: Parser> + 'a,
+ Op: Parser> + 'a,
+{
+ let term = term.map_with_span(|e, s| (e, s)).boxed();
+
+ (term.clone())
+ .then(op.then(term).repeated())
+ .foldl(|left, (op, right)| {
+ let span = left.1.start..right.1.end;
+ let kind = ExprKind::Binary {
+ left: Box::new(left.0),
+ op,
+ right: Box::new(right.0),
+ };
+ (into_expr(kind, span.clone()), span)
+ })
+ .map(|(e, _)| e)
+ .boxed()
+}
+
+fn func_call(expr: E) -> impl Parser>
+where
+ E: Parser> + Clone,
+{
+ let func = expr.clone();
+
+ let named_arg = ident_part()
+ .map(Some)
+ .then_ignore(ctrl(':'))
+ .then(expr.clone());
+
+ let assign_call =
+ ident_part()
+ .then_ignore(ctrl('='))
+ .then(expr.clone())
+ .map(|(alias, expr)| Expr {
+ alias: Some(alias),
+ ..expr
+ });
+ let positional_arg = assign_call.or(expr).map(|expr| (None, expr));
+
+ let args = named_arg.or(positional_arg).repeated();
+
+ func.then(args)
+ .validate(|(name, args), span, emit| {
+ if args.is_empty() {
+ return name.kind;
+ }
+
+ let mut named_args = HashMap::new();
+ let mut positional = Vec::new();
+ for (name, arg) in args {
+ if let Some(name) = name {
+ if named_args.contains_key(&name) {
+ let err = Simple::custom(span.clone(), "argument is used multiple times");
+ emit(err)
+ }
+ named_args.insert(name, arg);
+ } else {
+ positional.push(arg);
+ }
+ }
+
+ ExprKind::FuncCall(FuncCall {
+ name: Box::new(name),
+ args: positional,
+ named_args,
+ })
+ })
+ .map_with_span(into_expr)
+ .labelled("function call")
+}
+
+pub fn ident() -> impl Parser> {
+ let star = ctrl('*').to("*".to_string());
+
+ ident_part()
+ .chain(ctrl('.').ignore_then(ident_part().or(star)).repeated())
+ .map(Ident::from_path::)
+ .labelled("identifier")
+}
+
+fn operator_unary() -> impl Parser> {
+ (ctrl('+').to(UnOp::Add))
+ .or(ctrl('-').to(UnOp::Neg))
+ .or(ctrl('!').to(UnOp::Not))
+ .or(just(Token::Eq).to(UnOp::EqSelf))
+}
+fn operator_mul() -> impl Parser> {
+ (ctrl('*').to(BinOp::Mul))
+ .or(ctrl('/').to(BinOp::Div))
+ .or(ctrl('%').to(BinOp::Mod))
+}
+fn operator_add() -> impl Parser> {
+ (ctrl('+').to(BinOp::Add)).or(ctrl('-').to(BinOp::Sub))
+}
+fn operator_compare() -> impl Parser> {
+ (just(Token::Eq).to(BinOp::Eq))
+ .or(just(Token::Ne).to(BinOp::Ne))
+ .or(just(Token::Lte).to(BinOp::Lte))
+ .or(just(Token::Gte).to(BinOp::Gte))
+ .or(ctrl('<').to(BinOp::Lt))
+ .or(ctrl('>').to(BinOp::Gt))
+}
+fn operator_and() -> impl Parser> {
+ just(Token::And).to(BinOp::And)
+}
+pub fn operator_or() -> impl Parser> {
+ just(Token::Or).to(BinOp::Or)
+}
+fn operator_coalesce() -> impl Parser> {
+ just(Token::Coalesce).to(BinOp::Coalesce)
+}
diff --git a/prql-compiler/src/parser/interpolation.rs b/prql-compiler/src/parser/interpolation.rs
new file mode 100644
index 000000000000..2b3c75f92f5a
--- /dev/null
+++ b/prql-compiler/src/parser/interpolation.rs
@@ -0,0 +1,55 @@
+use chumsky::{error::Cheap, prelude::*};
+use itertools::Itertools;
+
+use crate::ast::pl::*;
+
+use super::{common::into_expr, lexer::*};
+
+/// Parses interpolated strings
+pub fn parse(
+ string: String,
+ span_offset: usize,
+) -> Result, Vec>> {
+ let res = parser(span_offset).parse(string);
+
+ match res {
+ Ok(items) => Ok(items),
+ Err(errors) => Err(errors
+ .into_iter()
+ .map(|err| {
+ Simple::expected_input_found(offset_span(err.span(), span_offset), None, None)
+ })
+ .collect_vec()),
+ }
+}
+
+fn parser(span_offset: usize) -> impl Parser, Error = Cheap> {
+ let expr = ident_part()
+ .separated_by(just('.'))
+ .delimited_by(just('{'), just('}'))
+ .map(Ident::from_path)
+ .map(ExprKind::Ident)
+ .map_with_span(move |e, s| into_expr(e, offset_span(s, span_offset)))
+ .map(Box::new)
+ .map(InterpolateItem::Expr);
+
+ let escape = (just("{{").to('{'))
+ .chain(just("}}").not().repeated())
+ .chain(just("}}").to('}'))
+ .collect::()
+ .map(InterpolateItem::String);
+
+ let string = none_of('{')
+ .repeated()
+ .at_least(1)
+ .collect::()
+ .map(InterpolateItem::String);
+
+ escape.or(expr).or(string).repeated().then_ignore(end())
+}
+
+fn offset_span(mut span: std::ops::Range, span_offset: usize) -> std::ops::Range {
+ span.start += span_offset;
+ span.end += span_offset;
+ span
+}
diff --git a/prql-compiler/src/parser/lexer.rs b/prql-compiler/src/parser/lexer.rs
new file mode 100644
index 000000000000..ed787d4dcf14
--- /dev/null
+++ b/prql-compiler/src/parser/lexer.rs
@@ -0,0 +1,420 @@
+use chumsky::{error::Cheap, prelude::*};
+
+use crate::ast::pl::*;
+
+#[derive(Clone, PartialEq, Debug)]
+pub enum Token {
+ NewLine,
+
+ Ident(String),
+ Keyword(String),
+ Literal(Literal),
+ Param(String),
+
+ Range {
+ bind_left: bool,
+ bind_right: bool,
+ },
+ Interpolation(char, String),
+
+ /// single-char control tokens
+ Control(char),
+
+ // TODO: rename to ArrowThin
+ Arrow, // ->
+ // TODO: rename to ArrowFat
+ ArrowDouble, // =>
+ Eq, // ==
+ Ne, // !=
+ Gte, // >=
+ Lte, // <=
+ And, // and
+ Or, // or
+ Coalesce, // ??
+}
+
+pub fn lexer() -> impl Parser)>, Error = Cheap> {
+ let new_line = just('\n').to(Token::NewLine);
+ let whitespace = one_of("\t \r").repeated().at_least(1).ignored();
+
+ let control_multi = choice((
+ just("->").to(Token::Arrow),
+ just("=>").to(Token::ArrowDouble),
+ just("==").to(Token::Eq),
+ just("!=").to(Token::Ne),
+ just(">=").to(Token::Gte),
+ just("<=").to(Token::Lte),
+ just("and").then_ignore(end_expr()).to(Token::And),
+ just("or").then_ignore(end_expr()).to(Token::Or),
+ just("??").to(Token::Coalesce),
+ ));
+
+ let control = one_of(">()
+ .map(Token::Param);
+
+ // s-string and f-strings
+ let interpolation = one_of("sf")
+ .then(quoted_string(true))
+ .map(|(c, s)| Token::Interpolation(c, s));
+
+ let token = choice((
+ new_line.clone(),
+ control_multi,
+ interpolation,
+ param,
+ control,
+ literal,
+ keyword,
+ ident,
+ ))
+ .recover_with(skip_then_retry_until([]).skip_start());
+
+ let comment = just('#').then(none_of('\n').repeated());
+ let comments = comment
+ .separated_by(new_line.then(whitespace.clone().or_not()))
+ .at_least(1)
+ .ignored();
+
+ let range = (whitespace.clone().or_not())
+ .then_ignore(just(".."))
+ .then(whitespace.clone().or_not())
+ .map(|(left, right)| Token::Range {
+ bind_left: left.is_none(),
+ bind_right: right.is_none(),
+ })
+ .map_with_span(|tok, span| (tok, span));
+
+ // range needs to consume leading whitespace,
+ // so whitespace following a token must not be consumed
+ let ignored = comments.clone().or(whitespace).repeated();
+
+ comments
+ .or_not()
+ .ignore_then(choice((
+ range,
+ ignored
+ .clone()
+ .ignore_then(token.map_with_span(|tok, span| (tok, span))),
+ )))
+ .repeated()
+ .then_ignore(ignored)
+ .then_ignore(end())
+}
+
+pub fn ident_part() -> impl Parser> {
+ let plain = filter(|c: &char| c.is_alphabetic() || *c == '_')
+ .map(Some)
+ .chain::, _>(filter(|c: &char| c.is_alphanumeric() || *c == '_').repeated())
+ .collect();
+
+ let backticks = just('`')
+ .ignore_then(none_of('`').repeated())
+ .then_ignore(just('`'))
+ .collect::();
+
+ plain.or(backticks)
+}
+
+fn literal() -> impl Parser> {
+ let exp = just('e').or(just('E')).ignore_then(
+ just('+')
+ .or(just('-'))
+ .or_not()
+ .chain::(text::digits(10)),
+ );
+
+ let integer = filter(|c: &char| c.is_ascii_digit() && *c != '0')
+ .chain::<_, Vec, _>(filter(|c: &char| c.is_ascii_digit() || *c == '_').repeated())
+ .or(just('0').map(|c| vec![c]));
+
+ let frac = just('.')
+ .chain::(filter(|c: &char| c.is_ascii_digit()))
+ .chain::(filter(|c: &char| c.is_ascii_digit() || *c == '_').repeated());
+
+ let number = just('+')
+ .or(just('-'))
+ .or_not()
+ .chain::(integer)
+ .chain::(frac.or_not().flatten())
+ .chain::(exp.or_not().flatten())
+ .try_map(|chars, span| {
+ let str = chars.into_iter().filter(|c| *c != '_').collect::();
+
+ if let Ok(i) = str.parse::() {
+ Ok(Literal::Integer(i))
+ } else if let Ok(f) = str.parse::() {
+ Ok(Literal::Float(f))
+ } else {
+ Err(Cheap::expected_input_found(span, None, None))
+ }
+ })
+ .labelled("number");
+
+ let string = quoted_string(true).map(Literal::String);
+
+ let raw_string = just("r")
+ .ignore_then(quoted_string(false))
+ .map(Literal::String);
+
+ let bool = (just("true").to(true))
+ .or(just("false").to(false))
+ .then_ignore(end_expr())
+ .map(Literal::Boolean);
+
+ let null = just("null").to(Literal::Null).then_ignore(end_expr());
+
+ let value_and_unit = integer
+ .then(choice((
+ just("microseconds"),
+ just("milliseconds"),
+ just("seconds"),
+ just("minutes"),
+ just("hours"),
+ just("days"),
+ just("weeks"),
+ just("months"),
+ just("years"),
+ )))
+ .then_ignore(end_expr())
+ .try_map(|(number, unit), span| {
+ let str = number.into_iter().filter(|c| *c != '_').collect::();
+ if let Ok(n) = str.parse::() {
+ let unit = unit.to_string();
+ Ok(ValueAndUnit { n, unit })
+ } else {
+ Err(Cheap::expected_input_found(span, None, None))
+ }
+ })
+ .map(Literal::ValueAndUnit);
+
+ let date_inner = digits(4)
+ .chain(just('-'))
+ .chain::(digits(2))
+ .chain::(just('-'))
+ .chain::(digits(2))
+ .boxed();
+
+ let time_inner = digits(2)
+ // minutes
+ .chain::(just(':').chain(digits(2)).or_not().flatten())
+ // seconds
+ .chain::(just(':').chain(digits(2)).or_not().flatten())
+ // milliseconds
+ .chain::(
+ just('.')
+ .chain(
+ filter(|c: &char| c.is_ascii_digit())
+ .repeated()
+ .at_least(1)
+ .at_most(6),
+ )
+ .or_not()
+ .flatten(),
+ )
+ // timezone offset
+ .chain::(
+ one_of("-+")
+ .chain(
+ // TODO: This is repeated without the `:`~ with an `or`
+ // because using `.or_not` triggers a request for
+ // type hints, which seems difficult to provide... Is there
+ // an easier way?
+ //
+ // (digits(2).chain(just(':').or_not()).chain(digits(2)))
+ //
+ (digits(2).chain(just(':')).chain(digits(2)))
+ .or(digits(2).chain(digits(2)))
+ .or(just('Z').map(|x| vec![x])),
+ )
+ .or_not()
+ .flatten(),
+ )
+ .boxed();
+
+ let date = just('@')
+ .ignore_then(date_inner.clone())
+ .then_ignore(end_expr())
+ .collect::()
+ .map(Literal::Date);
+
+ let time = just('@')
+ .ignore_then(time_inner.clone())
+ .then_ignore(end_expr())
+ .collect::()
+ .map(Literal::Time);
+
+ let datetime = just('@')
+ .ignore_then(date_inner)
+ .chain(just('T'))
+ .chain::(time_inner)
+ .then_ignore(end_expr())
+ .collect::()
+ .map(Literal::Timestamp);
+
+ choice((
+ string,
+ raw_string,
+ value_and_unit,
+ number,
+ bool,
+ null,
+ datetime,
+ date,
+ time,
+ ))
+}
+
+fn quoted_string(escaped: bool) -> impl Parser> {
+ // I don't know how this could be simplified and implemented for n>3 in general
+ choice((
+ quoted_string_inner(r#""""""""#, escaped),
+ quoted_string_inner(r#"""""""#, escaped),
+ quoted_string_inner(r#""""""#, escaped),
+ quoted_string_inner(r#"""""#, escaped),
+ quoted_string_inner(r#"""#, escaped),
+ quoted_string_inner(r#"''''''"#, escaped),
+ quoted_string_inner(r#"'''''"#, escaped),
+ quoted_string_inner(r#"''''"#, escaped),
+ quoted_string_inner(r#"'''"#, escaped),
+ quoted_string_inner(r#"'"#, escaped),
+ ))
+ .collect::()
+ .labelled("string")
+}
+
+fn quoted_string_inner(
+ quotes: &str,
+ escaping: bool,
+) -> impl Parser, Error = Cheap> + '_ {
+ let mut forbidden = just(quotes).boxed();
+
+ if escaping {
+ forbidden = just(quotes).or(just("\\")).boxed()
+ };
+
+ let mut inner = forbidden.not().boxed();
+
+ if escaping {
+ inner = inner
+ .or(just('\\').ignore_then(
+ just('\\')
+ .or(just('/'))
+ .or(just('"'))
+ .or(just('b').to('\x08'))
+ .or(just('f').to('\x0C'))
+ .or(just('n').to('\n'))
+ .or(just('r').to('\r'))
+ .or(just('t').to('\t'))
+ .or(just('u').ignore_then(
+ filter(|c: &char| c.is_ascii_hexdigit())
+ .repeated()
+ .exactly(4)
+ .collect::()
+ .validate(|digits, span, emit| {
+ char::from_u32(u32::from_str_radix(&digits, 16).unwrap())
+ .unwrap_or_else(|| {
+ emit(Cheap::expected_input_found(span, None, None));
+ '\u{FFFD}' // unicode replacement character
+ })
+ }),
+ )),
+ ))
+ .boxed();
+ }
+
+ inner.repeated().delimited_by(just(quotes), just(quotes))
+}
+
+fn digits(count: usize) -> impl Parser, Error = Cheap> {
+ filter(|c: &char| c.is_ascii_digit())
+ .repeated()
+ .exactly(count)
+}
+
+fn end_expr() -> impl Parser> {
+ choice((end(), one_of(",)]\r\n\t ").ignored(), just("..").ignored())).rewind()
+}
+
+impl Token {
+ pub fn range(bind_left: bool, bind_right: bool) -> Self {
+ Token::Range {
+ bind_left,
+ bind_right,
+ }
+ }
+}
+
+// This is here because Literal::Float(f64) does not implement Hash, so we cannot simply derive it.
+// There are reasons for that, but chumsky::Error needs Hash for the Token, so it can deduplicate
+// tokens in error.
+// So this hack could lead to duplicated tokens in error messages. Oh no.
+#[allow(clippy::derive_hash_xor_eq)]
+impl std::hash::Hash for Token {
+ fn hash(&self, state: &mut H) {
+ core::mem::discriminant(self).hash(state);
+ }
+}
+
+impl std::cmp::Eq for Token {}
+
+impl std::fmt::Display for Token {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::NewLine => write!(f, "new line"),
+ Self::Ident(arg0) => {
+ if arg0.is_empty() {
+ write!(f, "an identifier")
+ } else {
+ write!(f, "`{arg0}`")
+ }
+ }
+ Self::Keyword(arg0) => write!(f, "keyword {arg0}"),
+ Self::Literal(arg0) => write!(f, "{arg0}"),
+ Self::Control(arg0) => write!(f, "{arg0}"),
+
+ Self::Arrow => f.write_str("->"),
+ Self::ArrowDouble => f.write_str("=>"),
+ Self::Eq => f.write_str("=="),
+ Self::Ne => f.write_str("!="),
+ Self::Gte => f.write_str(">="),
+ Self::Lte => f.write_str("<="),
+ Self::And => f.write_str("and"),
+ Self::Or => f.write_str("or"),
+ Self::Coalesce => f.write_str("??"),
+
+ Self::Param(id) => write!(f, "${id}"),
+
+ Self::Range {
+ bind_left,
+ bind_right,
+ } => write!(
+ f,
+ "'{}..{}'",
+ if *bind_left { "" } else { " " },
+ if *bind_right { "" } else { " " }
+ ),
+ Self::Interpolation(c, s) => {
+ write!(f, "{c}\"{}\"", s)
+ }
+ }
+ }
+}
diff --git a/prql-compiler/src/parser/mod.rs b/prql-compiler/src/parser/mod.rs
index 12aad8385407..429c5fdd3b30 100644
--- a/prql-compiler/src/parser/mod.rs
+++ b/prql-compiler/src/parser/mod.rs
@@ -1,458 +1,213 @@
-//! This module contains the parser, which is responsible for converting a tree
-//! of pest pairs into a tree of AST Items. It has a small function to call into
-//! pest to get the parse tree / concrete syntax tree, and then a large
-//! function for turning that into PRQL AST.
-mod chumsky;
-
-use std::collections::HashMap;
-use std::str::FromStr;
-
-use anyhow::bail;
-use anyhow::{anyhow, Result};
+mod expr;
+mod interpolation;
+mod lexer;
+mod stmt;
+
+use anyhow::Result;
+use chumsky::{
+ error::{Cheap, SimpleReason},
+ prelude::*,
+ Stream,
+};
use itertools::Itertools;
-use pest::iterators::Pair;
-use pest::iterators::Pairs;
-use pest::Parser;
-use pest_derive::Parser;
-use super::ast::pl::*;
-use super::utils::*;
-use crate::error::Span;
+use self::lexer::Token;
-#[derive(Parser)]
-#[grammar = "parser/prql.pest"]
-struct PrqlParser;
+use super::ast::pl::*;
-pub(crate) type PestError = pest::error::Error;
-pub(crate) type PestRule = Rule;
+use crate::error::{Error, Errors, Reason};
/// Build PL AST from a PRQL query string.
-pub fn parse(string: &str) -> Result> {
- let pairs = parse_tree_of_str(string, Rule::statements)?;
-
- stmts_of_parse_pairs(pairs)
-}
-
-/// Parse a string into a parse tree / concrete syntax tree, made up of pest Pairs.
-fn parse_tree_of_str(source: &str, rule: Rule) -> Result> {
- Ok(PrqlParser::parse(rule, source)?)
-}
+pub fn parse(source: &str) -> Result> {
+ let mut errors = Vec::new();
-/// Parses a parse tree of pest Pairs into a list of statements.
-fn stmts_of_parse_pairs(pairs: Pairs) -> Result> {
- pairs
- .filter(|p| !matches!(p.as_rule(), Rule::EOI))
- .map(stmt_of_parse_pair)
- .collect()
-}
+ let (tokens, lex_errors) = ::chumsky::Parser::parse_recovery(&lexer::lexer(), source);
-fn stmt_of_parse_pair(pair: Pair) -> Result {
- let span = pair.as_span();
- let rule = pair.as_rule();
+ errors.extend(
+ lex_errors
+ .into_iter()
+ .map(|e| convert_lexer_error(source, e)),
+ );
- let kind = match rule {
- Rule::pipeline_stmt => {
- let pipeline = expr_of_parse_pair(pair.into_inner().next().unwrap())?;
- StmtKind::Main(Box::new(pipeline))
- }
- Rule::query_def => {
- let mut params: HashMap<_, _> = pair
- .into_inner()
- .map(|x| parse_named(x.into_inner()))
- .try_collect()?;
-
- let version = params
- .remove("version")
- .map(|v| v.try_cast(|i| i.parse_version(), None, "semver version number string"))
- .transpose()?;
-
- let other = params
- .into_iter()
- .flat_map(|(key, value)| match value.kind {
- ExprKind::Ident(value) => Some((key, value.to_string())),
- _ => None,
- })
- .collect();
-
- StmtKind::QueryDef(QueryDef { version, other })
- }
- Rule::func_def => {
- let mut pairs = pair.into_inner();
- let name = pairs.next().unwrap();
- let params = pairs.next().unwrap();
- let body = pairs.next().unwrap();
-
- let (name, return_type, _) = parse_typed_ident(name)?;
-
- let params: Vec<_> = params
- .into_inner()
- .into_iter()
- .map(parse_typed_ident)
- .try_collect()?;
-
- let mut positional_params = vec![];
- let mut named_params = vec![];
- for (name, ty, default_value) in params {
- let param = FuncParam {
- name,
- ty,
- default_value,
- };
- if param.default_value.is_some() {
- named_params.push(param)
- } else {
- positional_params.push(param)
- }
- }
+ let ast = if let Some(tokens) = tokens {
+ let len = source.chars().count();
+ let stream = Stream::from_iter(len..len + 1, tokens.into_iter());
- StmtKind::FuncDef(FuncDef {
- name,
- positional_params,
- named_params,
- body: Box::from(expr_of_parse_pair(body)?),
- return_ty: return_type,
- })
- }
- Rule::var_def => {
- let mut pairs = pair.into_inner();
+ let (ast, parse_errors) = ::chumsky::Parser::parse_recovery(&stmt::source(), stream);
- let name = parse_ident_part(pairs.next().unwrap());
- let value = Box::new(expr_of_parse_pair(pairs.next().unwrap())?);
+ errors.extend(parse_errors.into_iter().map(convert_parser_error));
- StmtKind::VarDef(VarDef { name, value })
- }
- _ => unreachable!("{pair}"),
+ ast
+ } else {
+ None
};
- let mut stmt = Stmt::from(kind);
- stmt.span = Some(Span {
- start: span.start(),
- end: span.end(),
- });
- Ok(stmt)
-}
-/// Parses a parse tree of pest Pairs into an AST.
-fn exprs_of_parse_pairs(pairs: Pairs) -> Result> {
- pairs
- .filter(|p| !matches!(p.as_rule(), Rule::EOI))
- .map(expr_of_parse_pair)
- .collect()
+ if errors.is_empty() {
+ Ok(ast.unwrap_or_default())
+ } else {
+ Err(Errors(errors).into())
+ }
}
-fn expr_of_parse_pair(pair: Pair) -> Result {
- let span = pair.as_span();
- let rule = pair.as_rule();
- let mut alias = None;
-
- let kind = match rule {
- Rule::list => ExprKind::List(exprs_of_parse_pairs(pair.into_inner())?),
- Rule::expr_mul | Rule::expr_add | Rule::expr_compare | Rule::expr => {
- let mut pairs = pair.into_inner();
-
- let mut expr = expr_of_parse_pair(pairs.next().unwrap())?;
- if let Some(op) = pairs.next() {
- let op = BinOp::from_str(op.as_str())?;
-
- expr = Expr::from(ExprKind::Binary {
- op,
- left: Box::new(expr),
- right: Box::new(expr_of_parse_pair(pairs.next().unwrap())?),
- });
- }
-
- expr.kind
- }
- Rule::expr_unary => {
- let mut pairs = pair.into_inner();
-
- let op = pairs.next().unwrap();
-
- let expr = expr_of_parse_pair(pairs.next().unwrap())?;
- let op = UnOp::from_str(op.as_str()).unwrap();
- match op {
- UnOp::Add => expr.kind,
- _ => ExprKind::Unary {
- op,
- expr: Box::new(expr),
- },
- }
- }
-
- // With coalesce, we need to grab the left and the right,
- // because we're transforming it into a function call rather
- // than passing along the operator. So this is unlike the rest
- // of the parsing (and maybe isn't optimal).
- Rule::expr_coalesce => {
- let mut pairs = pair.into_inner();
- let left = expr_of_parse_pair(pairs.next().unwrap())?;
-
- if pairs.next().is_none() {
- // If there's no coalescing, just return the single expression.
- left.kind
- } else {
- let right = expr_of_parse_pair(pairs.next().unwrap())?;
- ExprKind::Binary {
- left: Box::new(left),
- op: BinOp::Coalesce,
- right: Box::new(right),
- }
- }
- }
- // This makes the previous parsing a bit easier, but is hacky;
- // ideally find a better way (but it doesn't seem that easy to
- // parse parts of a Pairs).
- Rule::operator_coalesce => ExprKind::Ident(Ident::from_name("-")),
-
- Rule::assign | Rule::alias => {
- let (a, expr) = parse_named(pair.into_inner())?;
- alias = Some(a);
- expr.kind
- }
- Rule::func_call => {
- let mut pairs = pair.into_inner();
-
- let name = expr_of_parse_pair(pairs.next().unwrap())?;
-
- let mut named = HashMap::new();
- let mut positional = Vec::new();
- for arg in pairs {
- match arg.as_rule() {
- Rule::named_arg => {
- let (a, expr) = parse_named(arg.into_inner())?;
- named.insert(a, expr);
- }
- _ => {
- positional.push(expr_of_parse_pair(arg)?);
- }
- }
- }
+fn convert_lexer_error(source: &str, e: Cheap) -> Error {
+ // TODO: is there a neater way of taking a span? We want to take it based on
+ // the chars, not the bytes, so can't just index into the str.
+ let found = source
+ .chars()
+ .skip(e.span().start)
+ .take(e.span().end() - e.span().start)
+ .collect();
+ let span = common::into_span(e.span());
+
+ Error::new(Reason::Unexpected { found }).with_span(span)
+}
- ExprKind::FuncCall(FuncCall {
- name: Box::new(name),
- args: positional,
- named_args: named,
- })
- }
- Rule::jinja => {
- let inner = pair.as_str();
- ExprKind::Ident(Ident::from_name(inner))
- }
- Rule::ident => {
- // Pest has already parsed, so Chumsky should never fail
- let ident = ::chumsky::Parser::parse(&chumsky::ident(), pair.as_str()).unwrap();
+fn convert_parser_error(e: Simple) -> Error {
+ let span = common::into_span(e.span());
- ExprKind::Ident(ident)
- }
+ if let SimpleReason::Custom(message) = e.reason() {
+ return Error::new_simple(message).with_span(span);
+ }
- Rule::number => {
- // pest is responsible for ensuring these are in the correct place,
- // so we just need to remove them.
- let str = pair.as_str().replace('_', "");
-
- let lit = if let Ok(i) = str.parse::() {
- Literal::Integer(i)
- } else if let Ok(f) = str.parse::() {
- Literal::Float(f)
- } else {
- bail!("cannot parse {str} as number")
- };
- ExprKind::Literal(lit)
- }
- Rule::null => ExprKind::Literal(Literal::Null),
- Rule::boolean => ExprKind::Literal(Literal::Boolean(pair.as_str() == "true")),
- Rule::string => {
- // Takes the string_inner, without the quotes
- let inner = pair.into_inner().into_only();
- let inner = inner.map(|x| x.as_str().to_string()).unwrap_or_default();
- ExprKind::Literal(Literal::String(inner))
- }
- Rule::s_string => ExprKind::SString(ast_of_interpolate_items(pair)?),
- Rule::f_string => ExprKind::FString(ast_of_interpolate_items(pair)?),
- Rule::pipeline => {
- let mut nodes = exprs_of_parse_pairs(pair.into_inner())?;
- match nodes.len() {
- 0 => unreachable!(),
- 1 => nodes.remove(0).kind,
- _ => ExprKind::Pipeline(Pipeline { exprs: nodes }),
- }
- }
- Rule::nested_pipeline => {
- if let Some(pipeline) = pair.into_inner().next() {
- expr_of_parse_pair(pipeline)?.kind
- } else {
- ExprKind::Literal(Literal::Null)
+ let is_all_whitespace = e
+ .expected()
+ .all(|t| matches!(t, None | Some(Token::NewLine)));
+ let expecteds = e
+ .expected()
+ // TODO: could we collapse this into a `filter_map`? (though semantically
+ // identical)
+ //
+ // Only include whitespace if we're _only_ expecting whitespace
+ .filter(|t| is_all_whitespace || !matches!(t, None | Some(Token::NewLine)))
+ .map(|t| {
+ t.clone()
+ .map(|t| t.to_string())
+ .unwrap_or_else(|| "end of input".to_string())
+ })
+ .collect_vec();
+
+ let expected = if expecteds.is_empty() || expecteds.len() > 10 {
+ return Error::new(Reason::Unexpected {
+ found: e
+ .found()
+ .map(|c| c.to_string())
+ // I think a rare case where we have both no `expected` and no `found`.
+ // Would be good to know how often this happens; can improve if we are
+ // hitting it.
+ .unwrap_or_else(|| "end of input".to_string()),
+ })
+ .with_span(span);
+ } else {
+ let mut expecteds = expecteds;
+ expecteds.sort();
+
+ match expecteds.len() {
+ 1 => expecteds.remove(0),
+ 2 => expecteds.join(" or "),
+ _ => {
+ let last = expecteds.pop().unwrap();
+ format!("one of {} or {last}", expecteds.join(", "))
}
}
- Rule::range => {
- let [start, end]: [Option>; 2] = pair
- .into_inner()
- // Iterate over `start` & `end` (separator is not a term).
- .into_iter()
- .map(|x| {
- // Parse & Box each one.
- exprs_of_parse_pairs(x.into_inner())
- .and_then(|x| x.into_only())
- .map(Box::new)
- .ok()
- })
- .collect::>()
- .try_into()
- .map_err(|e| anyhow!("Expected start, separator, end; {e:?}"))?;
- ExprKind::Range(Range { start, end })
- }
-
- Rule::value_and_unit => {
- let pairs: Vec<_> = pair.into_inner().into_iter().collect();
- let [n, unit]: [Pair; 2] = pairs
- .try_into()
- .map_err(|e| anyhow!("Expected two items; {e:?}"))?;
-
- ExprKind::Literal(Literal::ValueAndUnit(ValueAndUnit {
- n: n.as_str().parse()?,
- unit: unit.as_str().to_owned(),
- }))
- }
-
- Rule::date | Rule::time | Rule::timestamp => {
- let inner = pair.into_inner().into_only()?.as_str().to_string();
-
- ExprKind::Literal(match rule {
- Rule::date => Literal::Date(inner),
- Rule::time => Literal::Time(inner),
- Rule::timestamp => Literal::Timestamp(inner),
- _ => unreachable!(),
- })
- }
-
- Rule::switch => {
- let cases = pair
- .into_inner()
- .map(|pair| -> Result<_> {
- let [condition, value]: [Expr; 2] = pair
- .into_inner()
- .map(expr_of_parse_pair)
- .collect::>>()?
- .try_into()
- .unwrap();
- Ok(SwitchCase { condition, value })
- })
- .try_collect()?;
-
- ExprKind::Switch(cases)
- }
-
- _ => unreachable!("{pair}"),
};
- Ok(Expr {
- span: Some(Span {
- start: span.start(),
- end: span.end(),
+
+ match e.found() {
+ Some(found) => Error::new(Reason::Expected {
+ who: e.label().map(|x| x.to_string()),
+ expected,
+ found: found.to_string(),
}),
- alias,
- ..Expr::from(kind)
- })
+ // We want a friendlier message than "found end of input"...
+ None => Error::new(Reason::Simple(format!(
+ "Expected {expected}, but didn't find anything before the end."
+ ))),
+ }
+ .with_span(span)
}
-fn type_of_parse_pair(pair: Pair