feat: support checking error message for statement & bump to 0.8.0

CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.0] - 2022-11-22
+
+- Support checking error message using `statement error <regex>` and `query error <regex>` syntax.
+  - Breaking change: `Record::Statement`, `Record::Query` and `TestErrorKind` are changed accordingly.
+
 ## [0.7.1] - 2022-11-15
 
 - Fix: `--external-engine-command-template` should not be required

Cargo.toml

@@ -2,7 +2,7 @@
 members = ["sqllogictest", "sqllogictest-bin", "examples/*", "tests"]
 
 [workspace.package]
-version = "0.7.1"
+version = "0.8.0"
 edition = "2021"
 homepage = "https://github.com/risinglightdb/sqllogictest-rs"
 keywords = ["sql", "database", "parser", "cli"]

README.md

@@ -16,7 +16,7 @@ Add the following lines to your `Cargo.toml` file:
 
 ```toml
 [dependencies]
-sqllogictest = "0.7"
+sqllogictest = "0.8"
 ```
 
 Implement `DB` trait for your database structure:

examples/basic/basic.slt

@@ -13,4 +13,26 @@ Bob
 Eve
 
 statement ok
-drop table t
+drop table t
+
+# The error message is: Hey you got FakeDBError!
+
+statement error
+give me an error
+
+statement error FakeDB
+give me an error
+
+statement error (?i)fakedb
+give me an error
+
+statement error Hey you
+give me an error
+
+# statement is expected to fail with error: "Hey we", but got error: "Hey you got FakeDBError!"
+# statement error Hey we
+# give me an error
+
+# query error is actually equivalent to statement error
+query error Hey you
+give me an error

examples/basic/examples/basic.rs

@@ -7,7 +7,7 @@ pub struct FakeDBError;
 
 impl std::fmt::Display for FakeDBError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:?}", self)
+        write!(f, "{:?}", "Hey you got FakeDBError!")
     }
 }
 
@@ -29,7 +29,7 @@ impl sqllogictest::DB for FakeDB {
         if sql.starts_with("drop") {
             return Ok("".into());
         }
-        unimplemented!("unsupported SQL: {}", sql);
+        Err(FakeDBError)
     }
 }
 

sqllogictest-bin/Cargo.toml

@@ -30,7 +30,7 @@ rand = "0.8"
 rust_decimal = { version = "1.7.0", features = ["tokio-pg"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-sqllogictest = { path = "../sqllogictest", version = "0.7" }
+sqllogictest = { path = "../sqllogictest", version = "0.8" }
 tempfile = "3"
 thiserror = "1"
 tokio = { version = "1", features = [

sqllogictest/Cargo.toml

@@ -19,3 +19,4 @@ tempfile = "3"
 thiserror = "1"
 futures = "0.3"
 libtest-mimic = "0.6"
+regex = "1.7.0"

sqllogictest/src/parser.rs

@@ -5,6 +5,8 @@ use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 
+use regex::Regex;
+
 use crate::ParseErrorKind::InvalidIncludeFile;
 
 /// The location in source file.
@@ -62,7 +64,7 @@ impl Location {
 }
 
 /// A single directive in a sqllogictest file.
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, Clone)]
 #[non_exhaustive]
 pub enum Record {
     /// An include copies all records from another files.
@@ -72,8 +74,9 @@ pub enum Record {
     Statement {
         loc: Location,
         conditions: Vec<Condition>,
-        /// The SQL command is expected to fail instead of to succeed.
-        error: bool,
+        /// The SQL command is expected to fail with an error messages that matches the given regex.
+        /// If the regex is an empty string, any error message is accepted.
+        expected_error: Option<Regex>,
         /// The SQL command.
         sql: String,
         /// Expected rows affected.
@@ -87,6 +90,9 @@ pub enum Record {
         type_string: String,
         sort_mode: Option<SortMode>,
         label: Option<String>,
+        /// The SQL command is expected to fail with an error messages that matches the given regex.
+        /// If the regex is an empty string, any error message is accepted.
+        expected_error: Option<Regex>,
         /// The SQL command.
         sql: String,
         /// The expected results.
@@ -203,6 +209,8 @@ pub enum ParseErrorKind {
     InvalidType(String),
     #[error("invalid number: {0:?}")]
     InvalidNumber(String),
+    #[error("invalid error message: {0:?}")]
+    InvalidErrorMessage(String),
     #[error("invalid duration: {0:?}")]
     InvalidDuration(String),
     #[error("invalid control: {0:?}")]
@@ -272,14 +280,19 @@ fn parse_inner(loc: &Location, script: &str) -> Result<Vec<Record>, ParseError>
             }
             ["statement", res @ ..] => {
                 let mut expected_count = None;
-                let error = match res {
-                    ["ok"] => false,
-                    ["error"] => true,
+                let mut expected_error = None;
+                match res {
+                    ["ok"] => {}
+                    ["error", err_str @ ..] => {
+                        let err_str = err_str.join(" ");
+                        expected_error = Some(Regex::new(&err_str).map_err(|_| {
+                            ParseErrorKind::InvalidErrorMessage(err_str).at(loc.clone())
+                        })?);
+                    }
                     ["count", count_str] => {
                         expected_count = Some(count_str.parse::<u64>().map_err(|_| {
                             ParseErrorKind::InvalidNumber((*count_str).into()).at(loc.clone())
                         })?);
-                        false
                     }
                     _ => return Err(ParseErrorKind::InvalidLine(line.into()).at(loc)),
                 };
@@ -297,17 +310,35 @@ fn parse_inner(loc: &Location, script: &str) -> Result<Vec<Record>, ParseError>
                 records.push(Record::Statement {
                     loc,
                     conditions: std::mem::take(&mut conditions),
-                    error,
+                    expected_error,
                     sql,
                     expected_count,
                 });
             }
-            ["query", type_string, res @ ..] => {
-                let sort_mode = match res.first().map(|&s| SortMode::try_from_str(s)).transpose() {
-                    Ok(sm) => sm,
-                    Err(k) => return Err(k.at(loc)),
-                };
-                let label = res.get(1).map(|s| s.to_string());
+            ["query", res @ ..] => {
+                let mut type_string = "";
+                let mut sort_mode = None;
+                let mut label = None;
+                let mut expected_error = None;
+                match res {
+                    ["error", err_str @ ..] => {
+                        let err_str = err_str.join(" ");
+                        expected_error = Some(Regex::new(&err_str).map_err(|_| {
+                            ParseErrorKind::InvalidErrorMessage(err_str).at(loc.clone())
+                        })?);
+                    }
+                    [type_str, res @ ..] => {
+                        type_string = type_str;
+                        sort_mode =
+                            match res.first().map(|&s| SortMode::try_from_str(s)).transpose() {
+                                Ok(sm) => sm,
+                                Err(k) => return Err(k.at(loc)),
+                            };
+                        label = res.get(1).map(|s| s.to_string());
+                    }
+                    _ => return Err(ParseErrorKind::InvalidLine(line.into()).at(loc)),
+                }
+
                 // The SQL for the query is found on second an subsequent lines of the record
                 // up to first line of the form "----" or until the end of the record.
                 let mut sql = match lines.next() {
@@ -345,6 +376,7 @@ fn parse_inner(loc: &Location, script: &str) -> Result<Vec<Record>, ParseError>
                     label,
                     sql,
                     expected_results,
+                    expected_error,
                 });
             }
             ["control", res @ ..] => match res {

sqllogictest/src/runner.rs

@@ -111,29 +111,47 @@ impl TestError {
     }
 }
 
+#[derive(Debug, Clone)]
+pub enum RecordKind {
+    Statement,
+    Query,
+}
+
+impl std::fmt::Display for RecordKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RecordKind::Statement => write!(f, "statement"),
+            RecordKind::Query => write!(f, "query"),
+        }
+    }
+}
+
 /// The error kind for running sqllogictest.
 #[derive(thiserror::Error, Debug, Clone)]
 pub enum TestErrorKind {
     #[error("parse error: {0}")]
     ParseError(ParseErrorKind),
-    #[error("statement is expected to fail, but actually succeed:\n[SQL] {sql}")]
-    StatementOk { sql: String },
-    #[error("statement failed: {err}\n[SQL] {sql}")]
-    StatementFail {
+    #[error("{kind} is expected to fail, but actually succeed:\n[SQL] {sql}")]
+    Ok { sql: String, kind: RecordKind },
+    #[error("{kind} failed: {err}\n[SQL] {sql}")]
+    Fail {
         sql: String,
         err: Arc<dyn std::error::Error + Send + Sync>,
+        kind: RecordKind,
+    },
+    #[error("{kind} is expected to fail with error:\n\t\x1b[91m{expected_err}\x1b[0m\nbut got error:\n\t\x1b[91m{err}\x1b[0m\n[SQL] {sql}")]
+    ErrorMismatch {
+        sql: String,
+        err: Arc<dyn std::error::Error + Send + Sync>,
+        expected_err: String,
+        kind: RecordKind,
     },
     #[error("statement is expected to affect {expected} rows, but actually {actual}\n[SQL] {sql}")]
     StatementResultMismatch {
         sql: String,
         expected: u64,
         actual: String,
     },
-    #[error("query failed: {err}\n[SQL] {sql}")]
-    QueryFail {
-        sql: String,
-        err: Arc<dyn std::error::Error + Send + Sync>,
-    },
     #[error("query result mismatch:\n[SQL] {sql}\n[Diff]\n{}", difference::Changeset::new(.expected, .actual, "\n"))]
     QueryResultMismatch {
         sql: String,
@@ -214,17 +232,24 @@ impl<D: AsyncDB> Runner<D> {
         match record {
             Record::Statement { conditions, .. } if self.should_skip(&conditions) => {}
             Record::Statement {
-                error,
+                conditions: _,
+
+                expected_error,
                 sql,
                 loc,
                 expected_count,
-                ..
             } => {
                 let sql = self.replace_keywords(sql);
                 let ret = self.db.run(&sql).await;
-                match ret {
-                    Ok(_) if error => return Err(TestErrorKind::StatementOk { sql }.at(loc)),
-                    Ok(count_str) => {
+                match (ret, expected_error) {
+                    (Ok(_), Some(_)) => {
+                        return Err(TestErrorKind::Ok {
+                            sql,
+                            kind: RecordKind::Statement,
+                        }
+                        .at(loc))
+                    }
+                    (Ok(count_str), None) => {
                         if let Some(expected_count) = expected_count {
                             if expected_count.to_string() != count_str {
                                 return Err(TestErrorKind::StatementResultMismatch {
@@ -236,38 +261,76 @@ impl<D: AsyncDB> Runner<D> {
                             }
                         }
                     }
-                    Err(e) if !error => {
-                        return Err(TestErrorKind::StatementFail {
+                    (Err(e), Some(expected_error)) => {
+                        if !expected_error.is_match(&e.to_string()) {
+                            return Err(TestErrorKind::ErrorMismatch {
+                                sql,
+                                err: Arc::new(e),
+                                expected_err: expected_error.to_string(),
+                                kind: RecordKind::Statement,
+                            }
+                            .at(loc));
+                        }
+                    }
+                    (Err(e), None) => {
+                        return Err(TestErrorKind::Fail {
                             sql,
                             err: Arc::new(e),
+                            kind: RecordKind::Statement,
                         }
                         .at(loc));
                     }
-                    _ => {}
                 }
                 if let Some(hook) = &mut self.hook {
                     hook.on_stmt_complete(&sql).await;
                 }
             }
             Record::Query { conditions, .. } if self.should_skip(&conditions) => {}
             Record::Query {
+                conditions: _,
+
                 loc,
                 sql,
+                expected_error,
                 expected_results,
                 sort_mode,
-                ..
+
+                // not handle yet,
+                type_string: _,
+                label: _,
             } => {
                 let sql = self.replace_keywords(sql);
-                let output = match self.db.run(&sql).await {
-                    Ok(output) => output,
-                    Err(e) => {
-                        return Err(TestErrorKind::QueryFail {
+                let output = match (self.db.run(&sql).await, expected_error) {
+                    (Ok(_), Some(_)) => {
+                        return Err(TestErrorKind::Ok {
+                            sql,
+                            kind: RecordKind::Query,
+                        }
+                        .at(loc))
+                    }
+                    (Ok(output), None) => output,
+                    (Err(e), Some(expected_error)) => {
+                        if !expected_error.is_match(&e.to_string()) {
+                            return Err(TestErrorKind::ErrorMismatch {
+                                sql,
+                                err: Arc::new(e),
+                                expected_err: expected_error.to_string(),
+                                kind: RecordKind::Query,
+                            }
+                            .at(loc));
+                        }
+                        return Ok(());
+                    }
+                    (Err(e), None) => {
+                        return Err(TestErrorKind::Fail {
                             sql,
                             err: Arc::new(e),
+                            kind: RecordKind::Query,
                         }
                         .at(loc));
                     }
                 };
+
                 let mut output = split_lines_and_normalize(&output);
                 let mut expected_results = split_lines_and_normalize(&expected_results);
                 match sort_mode.as_ref().or(self.sort_mode.as_ref()) {