From 11c1772d782c8fb4ed85b863d0d33c65dbebc57a Mon Sep 17 00:00:00 2001 From: taichong Date: Sat, 16 Apr 2022 09:34:44 +0800 Subject: [PATCH] compatibility mysql insert and select compatibility double quoted for MySQL: ``` insert into t values("xxx"); select * from t where col="xxx"; ``` --- .../test_stateless_cluster_linux/action.yml | 5 ++ .../test_stateless_cluster_macos/action.yml | 2 +- .../action.yml | 5 ++ .../action.yml | 2 +- Cargo.lock | 2 +- common/ast/Cargo.toml | 2 +- common/exception/Cargo.toml | 2 +- common/functions/Cargo.toml | 2 +- query/Cargo.toml | 2 +- query/src/servers/http/clickhouse_handler.rs | 56 ++++++++++--------- query/src/sessions/session_type.rs | 7 ++- query/src/sql/plan_parser.rs | 4 +- query/src/sql/sql_parser.rs | 30 +++++++--- query/src/sql/statements/analyzer_expr.rs | 16 ++++-- .../src/sql/statements/analyzer_value_expr.rs | 15 ++++- .../query/query_schema_joined_analyzer.rs | 3 +- query/src/sql/statements/value_source.rs | 30 +++++++--- query/tests/it/sql/sql_parser.rs | 11 ++-- .../sql/statements/query/query_normalizer.rs | 3 +- .../query/query_qualified_rewriter.rs | 3 +- .../query/query_schema_joined_analyzer.rs | 3 +- .../tests/it/sql/statements/statement_copy.rs | 3 +- .../sql/statements/statement_create_table.rs | 3 +- .../it/sql/statements/statement_select.rs | 3 +- .../00_dummy/00_0000_dummy_select_1.result | 6 +- .../05_ddl/05_0000_ddl_create_tables.sql | 3 + .../06_show/06_0004_show_tables.sql | 4 +- .../10_0000_python_clickhouse_driver.py | 44 +++++++++++++++ ...> 10_0000_python_clickhouse_driver.result} | 0 ...iver.py => 10_0000_python_mysql_driver.py} | 12 +++- .../10_0000_python_mysql_driver.result | 0 .../10_0001_python_sqlalchemy_driver.py | 10 +++- tools/fuzz/fuzz_targets/fuzz_parse_sql.rs | 4 +- 33 files changed, 223 insertions(+), 74 deletions(-) create mode 100755 tests/suites/0_stateless/10_drivers/10_0000_python_clickhouse_driver.py rename tests/suites/0_stateless/10_drivers/{10_0000_python_jdbc_driver.result => 10_0000_python_clickhouse_driver.result} (100%) rename tests/suites/0_stateless/10_drivers/{10_0000_python_jdbc_driver.py => 10_0000_python_mysql_driver.py} (73%) create mode 100644 tests/suites/0_stateless/10_drivers/10_0000_python_mysql_driver.result diff --git a/.github/actions/test_stateless_cluster_linux/action.yml b/.github/actions/test_stateless_cluster_linux/action.yml index 44bbe08c8f013..e424338a40077 100644 --- a/.github/actions/test_stateless_cluster_linux/action.yml +++ b/.github/actions/test_stateless_cluster_linux/action.yml @@ -28,6 +28,11 @@ runs: run: | chmod +x ./target/debug/databend-* + - name: Install python dependences + shell: bash + run: | + pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy clickhouse_driver + - name: Run Stateless Tests with Cluster mode shell: bash run: | diff --git a/.github/actions/test_stateless_cluster_macos/action.yml b/.github/actions/test_stateless_cluster_macos/action.yml index 0ca0769d03b43..0682c67347343 100644 --- a/.github/actions/test_stateless_cluster_macos/action.yml +++ b/.github/actions/test_stateless_cluster_macos/action.yml @@ -33,7 +33,7 @@ runs: - name: Install python dependences shell: bash run: | - pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy + pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy clickhouse_driver - name: Run Stateless Tests with Cluster mode shell: bash diff --git a/.github/actions/test_stateless_standalone_linux/action.yml b/.github/actions/test_stateless_standalone_linux/action.yml index a7e461a14b55e..4d9e6c879155f 100644 --- a/.github/actions/test_stateless_standalone_linux/action.yml +++ b/.github/actions/test_stateless_standalone_linux/action.yml @@ -28,6 +28,11 @@ runs: run: | chmod +x ./target/debug/databend-* + - name: Install python dependences + shell: bash + run: | + pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy clickhouse_driver + - name: Run Stateless Tests with Standalone mode, with embedded meta-store shell: bash run: | diff --git a/.github/actions/test_stateless_standalone_macos/action.yml b/.github/actions/test_stateless_standalone_macos/action.yml index 053e74e930c4e..7831fd811f3c5 100644 --- a/.github/actions/test_stateless_standalone_macos/action.yml +++ b/.github/actions/test_stateless_standalone_macos/action.yml @@ -28,7 +28,7 @@ runs: - name: Install python dependences shell: bash run: | - pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy + pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy clickhouse_driver - name: Set up file as executable shell: bash diff --git a/Cargo.lock b/Cargo.lock index 3f42e98e5b313..9a83fb818be3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5690,7 +5690,7 @@ dependencies = [ [[package]] name = "sqlparser" version = "0.13.1-alpha.0" -source = "git+https://github.com/datafuse-extras/sqlparser-rs?rev=f88fb32#f88fb320e8a1534b94e575fc6f1bcf44b30c6eaa" +source = "git+https://github.com/datafuse-extras/sqlparser-rs?rev=818c0f9#818c0f96a2dcc81537fbfe6c6ee9584600ebcb44" dependencies = [ "hashbrown 0.12.0", "log", diff --git a/common/ast/Cargo.toml b/common/ast/Cargo.toml index e26d73143e8f3..16901f52f5a31 100644 --- a/common/ast/Cargo.toml +++ b/common/ast/Cargo.toml @@ -19,7 +19,7 @@ common-functions = { path = "../functions" } # TODO (andylokandy): Use the version from crates.io once # https://github.com/brendanzab/codespan/pull/331 is released. codespan-reporting = { git = "https://github.com/brendanzab/codespan", rev = "c84116f5" } -sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "f88fb32" } +sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "818c0f9" } # Crates.io dependencies async-trait = "0.1.53" diff --git a/common/exception/Cargo.toml b/common/exception/Cargo.toml index e4660caaccdec..617de2c31aa18 100644 --- a/common/exception/Cargo.toml +++ b/common/exception/Cargo.toml @@ -26,4 +26,4 @@ tonic = "=0.6.2" # Github dependencies bincode = { git = "https://github.com/datafuse-extras/bincode", rev = "fd3f9ff" } -sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "f88fb32" } +sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "818c0f9" } diff --git a/common/functions/Cargo.toml b/common/functions/Cargo.toml index fab00ebb9e904..9b5c749afc92a 100644 --- a/common/functions/Cargo.toml +++ b/common/functions/Cargo.toml @@ -44,7 +44,7 @@ serde_json = "1.0.79" sha1 = "0.10.1" sha2 = "0.10.2" simdutf8 = "0.1.4" -sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "f88fb32" } +sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "818c0f9" } strength_reduce = "0.2.3" twox-hash = "1.6.2" uuid = { version = "0.8.2", features = ["v4"] } diff --git a/query/Cargo.toml b/query/Cargo.toml index 56c2e848b11b0..5b9ed6d6c3dac 100644 --- a/query/Cargo.toml +++ b/query/Cargo.toml @@ -54,7 +54,7 @@ common-tracing = { path = "../common/tracing" } bincode = { git = "https://github.com/datafuse-extras/bincode", rev = "fd3f9ff" } opensrv-clickhouse = { git = "https://github.com/datafuselabs/opensrv", rev = "9690be9", package = "opensrv-clickhouse" } opensrv-mysql = { git = "https://github.com/datafuselabs/opensrv", rev = "967477f1", package = "opensrv-mysql" } -sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "f88fb32" } +sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "818c0f9" } # Crates.io dependencies ahash = "0.7.6" diff --git a/query/src/servers/http/clickhouse_handler.rs b/query/src/servers/http/clickhouse_handler.rs index d247656b7371e..29610e7ffc56e 100644 --- a/query/src/servers/http/clickhouse_handler.rs +++ b/query/src/servers/http/clickhouse_handler.rs @@ -127,8 +127,11 @@ pub async fn clickhouse_handler_get( .map_err(InternalServerError) } -fn try_parse_insert_formatted(sql: &str) -> Result)>> { - if let Ok((statements, _)) = DfParser::parse_sql(sql) { +fn try_parse_insert_formatted( + sql: &str, + typ: SessionType, +) -> Result)>> { + if let Ok((statements, _)) = DfParser::parse_sql(sql, typ) { if statements.is_empty() { return Ok(None); } @@ -178,30 +181,33 @@ pub async fn clickhouse_handler_post( let mut sql = params.query; // Insert into format sql - let (plan, input_stream) = - if let Some((format, statements)) = try_parse_insert_formatted(&sql).map_err(BadRequest)? { - let plan = PlanParser::build_plan(statements, ctx.clone()) - .await - .map_err(InternalServerError)?; - ctx.attach_query_str(&sql); - - let input_stream = match format { - Format::NDJson => build_ndjson_stream(&plan, body).await.map_err(BadRequest)?, - }; - (plan, Some(input_stream)) - } else { - // Other sql - let body = body.into_string().await.map_err(BadRequest)?; - let sql = format!("{}\n{}", sql, body); - let (statements, _) = DfParser::parse_sql(&sql).map_err(BadRequest)?; - - let plan = PlanParser::build_plan(statements, ctx.clone()) - .await - .map_err(InternalServerError)?; - ctx.attach_query_str(&sql); - - (plan, None) + let (plan, input_stream) = if let Some((format, statements)) = + try_parse_insert_formatted(&sql, ctx.get_current_session().get_type()) + .map_err(BadRequest)? + { + let plan = PlanParser::build_plan(statements, ctx.clone()) + .await + .map_err(InternalServerError)?; + ctx.attach_query_str(&sql); + + let input_stream = match format { + Format::NDJson => build_ndjson_stream(&plan, body).await.map_err(BadRequest)?, }; + (plan, Some(input_stream)) + } else { + // Other sql + let body = body.into_string().await.map_err(BadRequest)?; + let sql = format!("{}\n{}", sql, body); + let (statements, _) = + DfParser::parse_sql(&sql, ctx.get_current_session().get_type()).map_err(BadRequest)?; + + let plan = PlanParser::build_plan(statements, ctx.clone()) + .await + .map_err(InternalServerError)?; + ctx.attach_query_str(&sql); + + (plan, None) + }; execute(ctx, plan, input_stream) .await diff --git a/query/src/sessions/session_type.rs b/query/src/sessions/session_type.rs index 5e3b5c5787eaf..759f2f03a7d54 100644 --- a/query/src/sessions/session_type.rs +++ b/query/src/sessions/session_type.rs @@ -24,11 +24,15 @@ pub enum SessionType { FlightRPC, HTTPAPI(String), Test, + Fuzz, } impl SessionType { pub fn is_user_session(&self) -> bool { - !matches!(self, SessionType::HTTPAPI(_) | SessionType::Test) + !matches!( + self, + SessionType::HTTPAPI(_) | SessionType::Test | SessionType::Fuzz + ) } } @@ -43,6 +47,7 @@ impl fmt::Display for SessionType { SessionType::Test => "Test".to_string(), SessionType::FlightRPC => "FlightRPC".to_string(), SessionType::HTTPAPI(usage) => format!("HTTPAPI({})", usage), + SessionType::Fuzz => "Fuzz".to_string(), }; write!(f, "{}", name) } diff --git a/query/src/sql/plan_parser.rs b/query/src/sql/plan_parser.rs index 81ea7119cf67d..9d34f9d3658d8 100644 --- a/query/src/sql/plan_parser.rs +++ b/query/src/sql/plan_parser.rs @@ -35,7 +35,7 @@ pub struct PlanParser; impl PlanParser { pub async fn parse(ctx: Arc, query: &str) -> Result { - let (statements, _) = DfParser::parse_sql(query)?; + let (statements, _) = DfParser::parse_sql(query, ctx.get_current_session().get_type())?; PlanParser::build_plan(statements, ctx).await } @@ -43,7 +43,7 @@ impl PlanParser { query: &str, ctx: Arc, ) -> (Result, Vec) { - match DfParser::parse_sql(query) { + match DfParser::parse_sql(query, ctx.get_current_session().get_type()) { Err(cause) => (Err(cause), vec![]), Ok((statements, hints)) => (PlanParser::build_plan(statements, ctx).await, hints), } diff --git a/query/src/sql/sql_parser.rs b/query/src/sql/sql_parser.rs index aab037075e9f3..a7d9d4027b5ec 100644 --- a/query/src/sql/sql_parser.rs +++ b/query/src/sql/sql_parser.rs @@ -24,6 +24,7 @@ use sqlparser::ast::Value; use sqlparser::dialect::keywords::Keyword; use sqlparser::dialect::Dialect; use sqlparser::dialect::GenericDialect; +use sqlparser::dialect::MySqlDialect; use sqlparser::dialect::SnowflakeDialect; use sqlparser::parser::Parser; use sqlparser::parser::ParserError; @@ -32,6 +33,7 @@ use sqlparser::tokenizer::Tokenizer; use sqlparser::tokenizer::Whitespace; use super::statements::DfShowRoles; +use crate::sessions::SessionType; use crate::sql::statements::DfShowEngines; use crate::sql::statements::DfShowMetrics; use crate::sql::statements::DfShowProcessList; @@ -57,7 +59,7 @@ pub struct DfParser<'a> { impl<'a> DfParser<'a> { /// Parse the specified tokens pub fn new(sql: &'a str) -> Result { - let dialect = &GenericDialect {}; + let dialect = &MySqlDialect {}; DfParser::new_with_dialect(sql, dialect) } @@ -73,12 +75,26 @@ impl<'a> DfParser<'a> { } /// Parse a SQL statement and produce a set of statements with dialect - pub fn parse_sql(sql: &'a str) -> Result<(Vec>, Vec), ErrorCode> { - let dialect = &GenericDialect {}; - let start = Instant::now(); - let result = DfParser::parse_sql_with_dialect(sql, dialect)?; - histogram!(super::metrics::METRIC_PARSER_USEDTIME, start.elapsed()); - Ok(result) + pub fn parse_sql( + sql: &'a str, + typ: SessionType, + ) -> Result<(Vec>, Vec), ErrorCode> { + match typ { + SessionType::MySQL => { + let dialect = &MySqlDialect {}; + let start = Instant::now(); + let result = DfParser::parse_sql_with_dialect(sql, dialect)?; + histogram!(super::metrics::METRIC_PARSER_USEDTIME, start.elapsed()); + Ok(result) + } + _ => { + let dialect = &GenericDialect {}; + let start = Instant::now(); + let result = DfParser::parse_sql_with_dialect(sql, dialect)?; + histogram!(super::metrics::METRIC_PARSER_USEDTIME, start.elapsed()); + Ok(result) + } + } } /// Parse a SQL statement and produce a set of statements diff --git a/query/src/sql/statements/analyzer_expr.rs b/query/src/sql/statements/analyzer_expr.rs index 38947d655dfa9..e6694b7a5b4a3 100644 --- a/query/src/sql/statements/analyzer_expr.rs +++ b/query/src/sql/statements/analyzer_expr.rs @@ -38,6 +38,7 @@ use sqlparser::ast::Value; use crate::procedures::ContextFunction; use crate::sessions::QueryContext; +use crate::sessions::SessionType; use crate::sql::statements::analyzer_value_expr::ValueExprAnalyzer; use crate::sql::statements::AnalyzableStatement; use crate::sql::statements::AnalyzedResult; @@ -61,7 +62,11 @@ impl ExpressionAnalyzer { // Build RPN for expr. Because async function unsupported recursion for rpn_item in &ExprRPNBuilder::build(self.context.clone(), expr).await? { match rpn_item { - ExprRPNItem::Value(v) => Self::analyze_value(v, &mut stack)?, + ExprRPNItem::Value(v) => Self::analyze_value( + v, + &mut stack, + self.context.get_current_session().get_type(), + )?, ExprRPNItem::Identifier(v) => self.analyze_identifier(v, &mut stack)?, ExprRPNItem::QualifiedIdentifier(v) => self.analyze_identifiers(v, &mut stack)?, ExprRPNItem::Function(v) => self.analyze_function(v, &mut stack)?, @@ -94,8 +99,8 @@ impl ExpressionAnalyzer { } } - fn analyze_value(value: &Value, args: &mut Vec) -> Result<()> { - args.push(ValueExprAnalyzer::analyze(value)?); + fn analyze_value(value: &Value, args: &mut Vec, typ: SessionType) -> Result<()> { + args.push(ValueExprAnalyzer::analyze(value, typ)?); Ok(()) } @@ -205,7 +210,10 @@ impl ExpressionAnalyzer { let mut parameters = Vec::with_capacity(info.parameters.len()); for parameter in &info.parameters { - match ValueExprAnalyzer::analyze(parameter)? { + match ValueExprAnalyzer::analyze( + parameter, + self.context.get_current_session().get_type(), + )? { Expression::Literal { value, .. } => { parameters.push(value); } diff --git a/query/src/sql/statements/analyzer_value_expr.rs b/query/src/sql/statements/analyzer_value_expr.rs index c79dc4d7df288..35318750ab946 100644 --- a/query/src/sql/statements/analyzer_value_expr.rs +++ b/query/src/sql/statements/analyzer_value_expr.rs @@ -19,16 +19,29 @@ use common_planners::Expression; use sqlparser::ast::DateTimeField; use sqlparser::ast::Value; +use crate::sessions::SessionType; + pub struct ValueExprAnalyzer; impl ValueExprAnalyzer { - pub fn analyze(value: &Value) -> Result { + pub fn analyze(value: &Value, typ: SessionType) -> Result { match value { Value::Null => Self::analyze_null_value(), Value::Boolean(value) => Self::analyze_bool_value(value), Value::Number(value, _) => Self::analyze_number_value(value, None), Value::HexStringLiteral(value) => Self::analyze_number_value(value, Some(16)), Value::SingleQuotedString(value) => Self::analyze_string_value(value), + Value::DoubleQuotedString(value) => { + if let SessionType::MySQL = typ { + Self::analyze_string_value(value) + } else { + Result::Err(ErrorCode::SyntaxException(format!( + "Unsupported value expression: {}, type: {:?}", + value, + Value::DoubleQuotedString(value.to_string()) + ))) + } + } Value::Interval { leading_precision: Some(_), .. diff --git a/query/src/sql/statements/query/query_schema_joined_analyzer.rs b/query/src/sql/statements/query/query_schema_joined_analyzer.rs index 495c4d5b9b759..ae933a8c93016 100644 --- a/query/src/sql/statements/query/query_schema_joined_analyzer.rs +++ b/query/src/sql/statements/query/query_schema_joined_analyzer.rs @@ -105,7 +105,8 @@ impl JoinedSchemaAnalyzer { if tbl_info.engine() == VIEW_ENGINE { if let Some(query) = tbl_info.options().get(QUERY) { - let (statements, _) = DfParser::parse_sql(query.as_str())?; + let (statements, _) = + DfParser::parse_sql(query.as_str(), self.ctx.get_current_session().get_type())?; if statements.len() == 1 { if let DfStatement::Query(subquery) = &statements[0] { if let AnalyzedResult::SelectQuery(state) = diff --git a/query/src/sql/statements/value_source.rs b/query/src/sql/statements/value_source.rs index 875578845cc85..0c87c0ac5f512 100644 --- a/query/src/sql/statements/value_source.rs +++ b/query/src/sql/statements/value_source.rs @@ -23,12 +23,14 @@ use common_io::prelude::*; use common_planners::Expression; use sqlparser::ast::Expr; use sqlparser::dialect::GenericDialect; +use sqlparser::dialect::MySqlDialect; use sqlparser::parser::Parser; use sqlparser::parser::ParserError; use sqlparser::tokenizer::Tokenizer; use crate::pipelines::transforms::ExpressionExecutor; use crate::sessions::QueryContext; +use crate::sessions::SessionType; use crate::sql::statements::ExpressionAnalyzer; pub struct ValueSource { @@ -103,7 +105,7 @@ impl ValueSource { analyzer: ExpressionAnalyzer, ctx: Arc, ) -> Result { - let values = parse_exprs(bytes)?; + let values = parse_exprs(bytes, ctx.get_current_session().get_type())?; let mut blocks = vec![]; for value in values { @@ -150,11 +152,23 @@ async fn exprs_to_datablock( executor.execute(&one_row_block) } -fn parse_exprs(buf: &[u8]) -> std::result::Result>, ParserError> { - let dialect = GenericDialect {}; - let sql = std::str::from_utf8(buf).unwrap(); - let mut tokenizer = Tokenizer::new(&dialect, sql); - let (tokens, position_map) = tokenizer.tokenize()?; - let mut parser = Parser::new(tokens, position_map, &dialect); - parser.parse_values() +fn parse_exprs(buf: &[u8], typ: SessionType) -> std::result::Result>, ParserError> { + match typ { + SessionType::MySQL => { + let dialect = MySqlDialect {}; + let sql = std::str::from_utf8(buf).unwrap(); + let mut tokenizer = Tokenizer::new(&dialect, sql); + let (tokens, position_map) = tokenizer.tokenize()?; + let mut parser = Parser::new(tokens, position_map, &dialect); + parser.parse_values() + } + _ => { + let dialect = GenericDialect {}; + let sql = std::str::from_utf8(buf).unwrap(); + let mut tokenizer = Tokenizer::new(&dialect, sql); + let (tokens, position_map) = tokenizer.tokenize()?; + let mut parser = Parser::new(tokens, position_map, &dialect); + parser.parse_values() + } + } } diff --git a/query/tests/it/sql/sql_parser.rs b/query/tests/it/sql/sql_parser.rs index f72922919cc14..9d2d885f3a2cc 100644 --- a/query/tests/it/sql/sql_parser.rs +++ b/query/tests/it/sql/sql_parser.rs @@ -13,6 +13,7 @@ // limitations under the License. use common_exception::Result; +use databend_query::sessions::SessionType; use databend_query::sql::statements::DfQueryStatement; use databend_query::sql::*; use pretty_assertions::assert_eq; @@ -23,7 +24,7 @@ use sqlparser::parser::ParserError; use sqlparser::tokenizer::Tokenizer; pub fn expect_parse_ok(sql: &str, expected: DfStatement) -> Result<()> { - let (statements, _) = DfParser::parse_sql(sql)?; + let (statements, _) = DfParser::parse_sql(sql, SessionType::Test)?; assert_eq!( statements.len(), 1, @@ -34,13 +35,13 @@ pub fn expect_parse_ok(sql: &str, expected: DfStatement) -> Result<()> { } pub fn expect_synonym_parse_eq(sql: &str, sql2: &str) -> Result<()> { - let (statements, _) = DfParser::parse_sql(sql)?; + let (statements, _) = DfParser::parse_sql(sql, SessionType::Test)?; assert_eq!( statements.len(), 1, "Expected to parse exactly one statement" ); - let (statements2, _) = DfParser::parse_sql(sql2)?; + let (statements2, _) = DfParser::parse_sql(sql2, SessionType::Test)?; assert_eq!( statements2.len(), 1, @@ -51,7 +52,7 @@ pub fn expect_synonym_parse_eq(sql: &str, sql2: &str) -> Result<()> { } pub fn expect_parse_err(sql: &str, expected: String) -> Result<()> { - let result = DfParser::parse_sql(sql); + let result = DfParser::parse_sql(sql, SessionType::Test); assert!(result.is_err(), "'{}' SHOULD BE '{}'", sql, expected); assert_eq!( result.unwrap_err().message(), @@ -64,7 +65,7 @@ pub fn expect_parse_err(sql: &str, expected: String) -> Result<()> { } pub fn expect_parse_err_contains(sql: &str, expected: String) -> Result<()> { - let result = DfParser::parse_sql(sql); + let result = DfParser::parse_sql(sql, SessionType::Test); assert!(result.is_err(), "'{}' SHOULD CONTAINS '{}'", sql, expected); assert!( result.unwrap_err().message().contains(&expected), diff --git a/query/tests/it/sql/statements/query/query_normalizer.rs b/query/tests/it/sql/statements/query/query_normalizer.rs index edd51f7013624..50f5173064c16 100644 --- a/query/tests/it/sql/statements/query/query_normalizer.rs +++ b/query/tests/it/sql/statements/query/query_normalizer.rs @@ -119,7 +119,8 @@ async fn test_query_normalizer() -> Result<()> { for test_case in &tests { let ctx = create_query_context().await?; - let (mut statements, _) = DfParser::parse_sql(test_case.query)?; + let (mut statements, _) = + DfParser::parse_sql(test_case.query, ctx.get_current_session().get_type())?; match statements.remove(0) { DfStatement::Query(query) => { diff --git a/query/tests/it/sql/statements/query/query_qualified_rewriter.rs b/query/tests/it/sql/statements/query/query_qualified_rewriter.rs index fda8e5147710b..363adc1a54e6b 100644 --- a/query/tests/it/sql/statements/query/query_qualified_rewriter.rs +++ b/query/tests/it/sql/statements/query/query_qualified_rewriter.rs @@ -101,7 +101,8 @@ async fn test_query_qualified_rewriter() -> Result<()> { for test_case in &tests { let ctx = create_query_context().await?; - let (mut statements, _) = DfParser::parse_sql(test_case.query)?; + let (mut statements, _) = + DfParser::parse_sql(test_case.query, ctx.get_current_session().get_type())?; match statements.remove(0) { DfStatement::Query(query) => { diff --git a/query/tests/it/sql/statements/query/query_schema_joined_analyzer.rs b/query/tests/it/sql/statements/query/query_schema_joined_analyzer.rs index d16772aed0697..aa2e3f1008441 100644 --- a/query/tests/it/sql/statements/query/query_schema_joined_analyzer.rs +++ b/query/tests/it/sql/statements/query/query_schema_joined_analyzer.rs @@ -53,7 +53,8 @@ async fn test_joined_schema_analyzer() -> Result<()> { for test_case in &tests { let ctx = create_query_context().await?; - let (mut statements, _) = DfParser::parse_sql(test_case.query)?; + let (mut statements, _) = + DfParser::parse_sql(test_case.query, ctx.get_current_session().get_type())?; match statements.remove(0) { DfStatement::Query(query) => { diff --git a/query/tests/it/sql/statements/statement_copy.rs b/query/tests/it/sql/statements/statement_copy.rs index 8572410052bbe..3901d8ff693ce 100644 --- a/query/tests/it/sql/statements/statement_copy.rs +++ b/query/tests/it/sql/statements/statement_copy.rs @@ -156,7 +156,8 @@ async fn test_statement_copy() -> Result<()> { for test in &tests { let ctx = create_query_context().await?; - let (mut statements, _) = DfParser::parse_sql(test.query)?; + let (mut statements, _) = + DfParser::parse_sql(test.query, ctx.get_current_session().get_type())?; let statement = statements.remove(0); if test.err.is_empty() { let result = statement.analyze(ctx).await?; diff --git a/query/tests/it/sql/statements/statement_create_table.rs b/query/tests/it/sql/statements/statement_create_table.rs index 6463ee381ea3c..9899aaf6ea8bb 100644 --- a/query/tests/it/sql/statements/statement_create_table.rs +++ b/query/tests/it/sql/statements/statement_create_table.rs @@ -27,7 +27,8 @@ async fn test_statement_create_table_reserved_opt_keys() -> Result<()> { let ctx = create_query_context().await?; for opt in &*RESERVED_TABLE_OPTION_KEYS { let query = format!("CREATE TABLE default.a( c int) {opt}= 1"); - let (mut statements, _) = DfParser::parse_sql(query.as_str())?; + let (mut statements, _) = + DfParser::parse_sql(query.as_str(), ctx.get_current_session().get_type())?; match statements.remove(0) { DfStatement::CreateTable(query) => match query.analyze(ctx.clone()).await { Err(e) => { diff --git a/query/tests/it/sql/statements/statement_select.rs b/query/tests/it/sql/statements/statement_select.rs index 0bbdf3ba26e82..d23ddc5f85d05 100644 --- a/query/tests/it/sql/statements/statement_select.rs +++ b/query/tests/it/sql/statements/statement_select.rs @@ -125,7 +125,8 @@ async fn test_statement_select_analyze() -> Result<()> { for test_case in &tests { let ctx = create_query_context().await?; - let (mut statements, _) = DfParser::parse_sql(test_case.query)?; + let (mut statements, _) = + DfParser::parse_sql(test_case.query, ctx.get_current_session().get_type())?; match statements.remove(0) { DfStatement::Query(query) => { diff --git a/tests/suites/0_stateless/00_dummy/00_0000_dummy_select_1.result b/tests/suites/0_stateless/00_dummy/00_0000_dummy_select_1.result index bf5c4fd68bee8..3915d3b9f12d0 100644 --- a/tests/suites/0_stateless/00_dummy/00_0000_dummy_select_1.result +++ b/tests/suites/0_stateless/00_dummy/00_0000_dummy_select_1.result @@ -14,7 +14,7 @@ a 0 1 2 -0 number 0 -1 number 1 -2 number 2 +0 number number +1 number number +2 number number That's good. diff --git a/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql b/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql index 1f3fd65e4ede1..2b8eab4aaad03 100644 --- a/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql +++ b/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql @@ -1,4 +1,7 @@ DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; CREATE TABLE t(c1 int) ENGINE = Null; SELECT COUNT(1) from system.tables where name = 't' and database = 'default'; diff --git a/tests/suites/0_stateless/06_show/06_0004_show_tables.sql b/tests/suites/0_stateless/06_show/06_0004_show_tables.sql index fe426e785f493..065280d7eced7 100644 --- a/tests/suites/0_stateless/06_show/06_0004_show_tables.sql +++ b/tests/suites/0_stateless/06_show/06_0004_show_tables.sql @@ -9,6 +9,8 @@ use showtable; SHOW TABLES; SHOW TABLES LIKE 't%'; +-- if want to support SHOW TABLES LIKE "t2" link to this pr: +-- https://github.com/datafuse-extras/sqlparser-rs/pull/34/files SHOW TABLES LIKE 't2'; SHOW TABLES LIKE 't'; @@ -19,7 +21,7 @@ SHOW TABLES WHERE table_name = 't2' AND 1 = 1; USE default; SHOW TABLES FROM showtables WHERE table_name LIKE 't%'; -SHOW TABLES FROM showtables WHERE table_name = 't%' AND 1 = 0; +SHOW TABLES FROM showtables WHERE table_name = "t%" AND 1 = 0; SHOW TABLES FROM showtables WHERE table_name = 't2' OR 1 = 1; SHOW TABLES FROM showtables WHERE table_name = 't2' AND 1 = 1; DROP DATABASE showtable; diff --git a/tests/suites/0_stateless/10_drivers/10_0000_python_clickhouse_driver.py b/tests/suites/0_stateless/10_drivers/10_0000_python_clickhouse_driver.py new file mode 100755 index 0000000000000..5ae25c4a2d472 --- /dev/null +++ b/tests/suites/0_stateless/10_drivers/10_0000_python_clickhouse_driver.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +from clickhouse_driver import connect +from clickhouse_driver.dbapi import OperationalError + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, '../../../helpers')) + +from client import client + +log = None +# uncomment the line below for debugging +log = sys.stdout + +client1 = client(name='client1>', log=log) + +sqls = """ +DROP DATABASE IF EXISTS db1; +CREATE DATABASE db1; +USE db1; +CREATE TABLE IF NOT EXISTS t1(a String, b String, c String, d String, e String, f String, g String, h String) Engine = Memory; +""" + +client1.run(sqls) +client = connect(host='127.0.0.1', database='db1', user='root', password='', port='9000') +cur = client.cursor() +try: + cur.execute('INSERT INTO db1.t1(a) VALUES("Test")') +except OperationalError as e: + assert ('DB:Exception. Unable to get field named "Test". Valid fields: ["a"].' in str(e)) +try: + res=cur.execute('SELECT a FROM db1.t1 WHERE a="Test"') + for t in res: + print(" ROW: {0}: {1}".format(type(t), t)) + for v in t: + print(" COLUMN: {0}: {1}".format(type(v), v)) +except Exception as e: + assert ('DB:Exception. Unknown column Test.' in str(e)) +finally: + sql = "DROP DATABASE db1;" + client1.run(sql) diff --git a/tests/suites/0_stateless/10_drivers/10_0000_python_jdbc_driver.result b/tests/suites/0_stateless/10_drivers/10_0000_python_clickhouse_driver.result similarity index 100% rename from tests/suites/0_stateless/10_drivers/10_0000_python_jdbc_driver.result rename to tests/suites/0_stateless/10_drivers/10_0000_python_clickhouse_driver.result diff --git a/tests/suites/0_stateless/10_drivers/10_0000_python_jdbc_driver.py b/tests/suites/0_stateless/10_drivers/10_0000_python_mysql_driver.py similarity index 73% rename from tests/suites/0_stateless/10_drivers/10_0000_python_jdbc_driver.py rename to tests/suites/0_stateless/10_drivers/10_0000_python_mysql_driver.py index 22864868dd55b..5ed5f09f27b32 100755 --- a/tests/suites/0_stateless/10_drivers/10_0000_python_jdbc_driver.py +++ b/tests/suites/0_stateless/10_drivers/10_0000_python_mysql_driver.py @@ -24,6 +24,11 @@ """ client1.run(sqls) + +sql1 = "INSERT INTO db1.t1(a) VALUES(%s);" % ('\"Test Some Inser\\"\'`ts\"') +client1.run(sql1) +sql2 = "INSERT INTO db1.t1(a) VALUES(%s);" % ("'Test Some Inser\"\\'`ts'") +client1.run(sql2) time.sleep(2) mydb = mysql.connector.connect(host="127.0.0.1", user="root", @@ -36,6 +41,11 @@ mycursor.execute("SHOW FULL TABLES FROM db1") res = mycursor.fetchall() assert res == [('t1', 'BASE TABLE')] - +sql3 = "SELECT COUNT(*) FROM db1.t1 WHERE a = %s" % ("\"Test Some Inser\\\"'`ts\"") +mycursor.execute(sql3) +res = mycursor.fetchall() +for row in res: + cnt = row[0] + assert 2 == cnt sql = "DROP DATABASE db1;" client1.run(sql) diff --git a/tests/suites/0_stateless/10_drivers/10_0000_python_mysql_driver.result b/tests/suites/0_stateless/10_drivers/10_0000_python_mysql_driver.result new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/suites/0_stateless/10_drivers/10_0001_python_sqlalchemy_driver.py b/tests/suites/0_stateless/10_drivers/10_0001_python_sqlalchemy_driver.py index 302c26f098420..a347ab0b23df4 100755 --- a/tests/suites/0_stateless/10_drivers/10_0001_python_sqlalchemy_driver.py +++ b/tests/suites/0_stateless/10_drivers/10_0001_python_sqlalchemy_driver.py @@ -1,8 +1,16 @@ #!/usr/bin/env python3 import sqlalchemy +import os -engine = sqlalchemy.create_engine("mysql+pymysql://root:root@localhost:3307/") +tcp_port = os.getenv("QUERY_MYSQL_HANDLER_PORT") +if tcp_port is None: + port = "3307" +else: + port = tcp_port + +uri = "mysql+pymysql://root:root@localhost:" + port + "/" +engine = sqlalchemy.create_engine(uri) conn = engine.connect() conn.execute("create database if not exists book_db") conn.execute("use book_db") diff --git a/tools/fuzz/fuzz_targets/fuzz_parse_sql.rs b/tools/fuzz/fuzz_targets/fuzz_parse_sql.rs index 611c94088a709..66046dfead3a9 100644 --- a/tools/fuzz/fuzz_targets/fuzz_parse_sql.rs +++ b/tools/fuzz/fuzz_targets/fuzz_parse_sql.rs @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_query::sessions::Session; +use databend_query::sessions::SessionType; use databend_query::sql::DfParser; use honggfuzz::fuzz; fn main() { loop { fuzz!(|data: String| { - let _ = DfParser::parse_sql(&data); + let _ = DfParser::parse_sql(&data, SessionType::Fuzz); }); } }