diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 8576d7277..d7e1a903b 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -25,6 +25,7 @@ use sqlparser_derive::{Visit, VisitMut}; use crate::ast::value::escape_single_quote_string; use crate::ast::{display_comma_separated, display_separated, DataType, Expr, Ident, ObjectName}; +use crate::location::Located; use crate::tokenizer::Token; /// An `ALTER TABLE` (`Statement::AlterTable`) operation @@ -46,12 +47,12 @@ pub enum AlterTableOperation { /// `DROP CONSTRAINT [ IF EXISTS ] ` DropConstraint { if_exists: bool, - name: Ident, + name: Located, cascade: bool, }, /// `DROP [ COLUMN ] [ IF EXISTS ] [ CASCADE ]` DropColumn { - column_name: Ident, + column_name: Located, if_exists: bool, cascade: bool, }, @@ -75,25 +76,28 @@ pub enum AlterTableOperation { }, /// `RENAME [ COLUMN ] TO ` RenameColumn { - old_column_name: Ident, - new_column_name: Ident, + old_column_name: Located, + new_column_name: Located, }, /// `RENAME TO ` RenameTable { table_name: ObjectName }, // CHANGE [ COLUMN ] [ ] ChangeColumn { - old_name: Ident, - new_name: Ident, + old_name: Located, + new_name: Located, data_type: DataType, options: Vec, }, /// `RENAME CONSTRAINT TO ` /// /// Note: this is a PostgreSQL-specific operation. - RenameConstraint { old_name: Ident, new_name: Ident }, + RenameConstraint { + old_name: Located, + new_name: Located, + }, /// `ALTER [ COLUMN ]` AlterColumn { - column_name: Ident, + column_name: Located, op: AlterColumnOperation, }, } @@ -272,8 +276,8 @@ impl fmt::Display for AlterColumnOperation { pub enum TableConstraint { /// `[ CONSTRAINT ] { PRIMARY KEY | UNIQUE } ()` Unique { - name: Option, - columns: Vec, + name: Option>, + columns: Vec>, /// Whether this is a `PRIMARY KEY` or just a `UNIQUE` constraint is_primary: bool, }, @@ -283,16 +287,16 @@ pub enum TableConstraint { /// [ON UPDATE ] [ON DELETE ] /// }`). ForeignKey { - name: Option, - columns: Vec, + name: Option>, + columns: Vec>, foreign_table: ObjectName, - referred_columns: Vec, + referred_columns: Vec>, on_delete: Option, on_update: Option, }, /// `[ CONSTRAINT ] CHECK ()` Check { - name: Option, + name: Option>, expr: Box, }, /// MySQLs [index definition][1] for index creation. Not present on ANSI so, for now, the usage @@ -305,13 +309,13 @@ pub enum TableConstraint { /// Whether this index starts with KEY (true) or INDEX (false), to maintain the same syntax. display_as_key: bool, /// Index name. - name: Option, + name: Option>, /// Optional [index type][1]. /// /// [1]: IndexType index_type: Option, /// Referred column identifier list. - columns: Vec, + columns: Vec>, }, /// MySQLs [fulltext][1] definition. Since the [`SPATIAL`][2] definition is exactly the same, /// and MySQL displays both the same way, it is part of this definition as well. @@ -332,9 +336,9 @@ pub enum TableConstraint { /// Whether the type is followed by the keyword `KEY`, `INDEX`, or no keyword at all. index_type_display: KeyOrIndexDisplay, /// Optional index name. - opt_index_name: Option, + opt_index_name: Option>, /// Referred column identifier list. - columns: Vec, + columns: Vec>, }, } @@ -490,7 +494,7 @@ impl fmt::Display for IndexType { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct ColumnDef { - pub name: Ident, + pub name: Located, pub data_type: DataType, pub collation: Option, pub options: Vec, @@ -526,7 +530,7 @@ impl fmt::Display for ColumnDef { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct ColumnOptionDef { - pub name: Option, + pub name: Option>, pub option: ColumnOption, } @@ -559,7 +563,7 @@ pub enum ColumnOption { /// }`). ForeignKey { foreign_table: ObjectName, - referred_columns: Vec, + referred_columns: Vec>, on_delete: Option, on_update: Option, }, @@ -611,8 +615,8 @@ impl fmt::Display for ColumnOption { } } -fn display_constraint_name(name: &'_ Option) -> impl fmt::Display + '_ { - struct ConstraintName<'a>(&'a Option); +fn display_constraint_name(name: &'_ Option>) -> impl fmt::Display + '_ { + struct ConstraintName<'a>(&'a Option>); impl<'a> fmt::Display for ConstraintName<'a> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Some(name) = self.0 { diff --git a/src/ast/mod.rs b/src/ast/mod.rs index b8d5cf042..411d4c73e 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -43,6 +43,7 @@ pub use self::value::{ escape_quoted_string, DateTimeField, DollarQuotedString, TrimWhereField, Value, }; +pub use crate::location::{Located, Location, Range}; #[cfg(feature = "visitor")] pub use visitor::*; @@ -107,7 +108,7 @@ pub struct Ident { impl Ident { /// Create a new identifier with the given value and no quotes. - pub fn new(value: S) -> Self + pub fn new_unlocated(value: S) -> Self where S: Into, { @@ -117,9 +118,24 @@ impl Ident { } } + /// Create a new identifier with the given value and no quotes. + pub fn new(value: S) -> Located + where + S: Into, + { + Located::new(Ident::new_unlocated(value), None) + } + + pub fn new_located(value: S, range: Range) -> Located + where + S: Into, + { + Located::new(Ident::new_unlocated(value), range.into_option()) + } + /// Create a new quoted identifier with the given quote and value. This function /// panics if the given quote is not a valid quote character. - pub fn with_quote(quote: char, value: S) -> Self + pub fn with_quote_unlocated(quote: char, value: S) -> Self where S: Into, { @@ -129,6 +145,27 @@ impl Ident { quote_style: Some(quote), } } + + pub fn with_quote(quote: char, value: S) -> Located + where + S: Into, + { + Located::new(Ident::with_quote_unlocated(quote, value), None) + } + + pub fn with_quote_located(quote: char, value: S, range: Range) -> Located + where + S: Into, + { + Located::new( + Ident::with_quote_unlocated(quote, value), + range.into_option(), + ) + } + + pub fn into_located(self, range: Range) -> Located { + Located::new(self, range.into_option()) + } } impl From<&str> for Ident { @@ -140,6 +177,18 @@ impl From<&str> for Ident { } } +impl From<&str> for Located { + fn from(value: &str) -> Self { + Located::new( + Ident { + value: value.to_string(), + quote_style: None, + }, + None, + ) + } +} + impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self.quote_style { @@ -158,7 +207,7 @@ impl fmt::Display for Ident { #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] -pub struct ObjectName(pub Vec); +pub struct ObjectName(pub Vec>); impl fmt::Display for ObjectName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -264,9 +313,9 @@ impl fmt::Display for JsonOperator { )] pub enum Expr { /// Identifier e.g. table name or column name - Identifier(Ident), + Identifier(Located), /// Multi-part identifier, e.g. `table_alias.column` or `schema.table.col` - CompoundIdentifier(Vec), + CompoundIdentifier(Vec>), /// JSON access (postgres) eg: data->'tags' JsonAccess { left: Box, @@ -274,7 +323,10 @@ pub enum Expr { right: Box, }, /// CompositeAccess (postgres) eg: SELECT (information_schema._pg_expandarray(array['i','i'])).n - CompositeAccess { expr: Box, key: Ident }, + CompositeAccess { + expr: Box, + key: Located, + }, /// `IS FALSE` operator IsFalse(Box), /// `IS NOT FALSE` operator @@ -518,7 +570,7 @@ pub enum Expr { /// [(1)]: https://dev.mysql.com/doc/refman/8.0/en/fulltext-search.html#function_match MatchAgainst { /// `(, , ...)`. - columns: Vec, + columns: Vec>, /// ``. match_value: Value, /// `` @@ -1114,7 +1166,7 @@ pub enum Statement { table_name: ObjectName, partitions: Option>, for_columns: bool, - columns: Vec, + columns: Vec>, cache_metadata: bool, noscan: bool, compute_statistics: bool, @@ -1144,7 +1196,7 @@ pub enum Statement { #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] table_name: ObjectName, /// COLUMNS - columns: Vec, + columns: Vec>, /// Overwrite (Hive) overwrite: bool, /// A SQL query that specifies what to insert @@ -1152,7 +1204,7 @@ pub enum Statement { /// partitioned insert (Hive) partitioned: Option>, /// Columns defined after PARTITION - after_columns: Vec, + after_columns: Vec>, /// whether the insert has the table keyword (Hive) table: bool, on: Option, @@ -1172,7 +1224,7 @@ pub enum Statement { #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] table_name: ObjectName, /// COLUMNS - columns: Vec, + columns: Vec>, /// If true, is a 'COPY TO' statement. If false is a 'COPY FROM' to: bool, /// The source of 'COPY FROM', or the target of 'COPY TO' @@ -1219,10 +1271,10 @@ pub enum Statement { materialized: bool, /// View name name: ObjectName, - columns: Vec, + columns: Vec>, query: Box, with_options: Vec, - cluster_by: Vec, + cluster_by: Vec>, }, /// CREATE TABLE CreateTable { @@ -1260,8 +1312,8 @@ pub enum Statement { #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] name: ObjectName, if_not_exists: bool, - module_name: Ident, - module_args: Vec, + module_name: Located, + module_args: Vec>, }, /// CREATE INDEX CreateIndex { @@ -1269,7 +1321,7 @@ pub enum Statement { name: ObjectName, #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] table_name: ObjectName, - using: Option, + using: Option>, columns: Vec, unique: bool, if_not_exists: bool, @@ -1290,11 +1342,11 @@ pub enum Statement { replication: Option, connection_limit: Option, valid_until: Option, - in_role: Vec, - in_group: Vec, - role: Vec, - user: Vec, - admin: Vec, + in_role: Vec>, + in_group: Vec>, + role: Vec>, + user: Vec>, + admin: Vec>, // MSSQL authorization_owner: Option, }, @@ -1341,7 +1393,7 @@ pub enum Statement { /// but may also compatible with other SQL. Declare { /// Cursor name - name: Ident, + name: Located, /// Causes the cursor to return data in binary rather than in text format. binary: bool, /// None = Not specified @@ -1364,7 +1416,7 @@ pub enum Statement { /// but may also compatible with other SQL. Fetch { /// Cursor name - name: Ident, + name: Located, direction: FetchDirection, /// Optional, It's possible to fetch rows form cursor to the table into: Option, @@ -1384,7 +1436,7 @@ pub enum Statement { /// Non-ANSI optional identifier to inform if the role is defined inside the current session (`SESSION`) or transaction (`LOCAL`). context_modifier: ContextModifier, /// Role name. If NONE is specified, then the current role name is removed. - role_name: Option, + role_name: Option>, }, /// ```sql /// SET @@ -1426,7 +1478,7 @@ pub enum Statement { /// ``` /// /// Note: this is a PostgreSQL-specific statement. - ShowVariable { variable: Vec }, + ShowVariable { variable: Vec> }, /// SHOW VARIABLES /// /// Note: this is a MySQL-specific statement. @@ -1454,7 +1506,7 @@ pub enum Statement { ShowTables { extended: bool, full: bool, - db_name: Option, + db_name: Option>, filter: Option, }, /// SHOW COLLATION @@ -1464,7 +1516,7 @@ pub enum Statement { /// USE /// /// Note: This is a MySQL-specific statement. - Use { db_name: Ident }, + Use { db_name: Located }, /// `{ BEGIN [ TRANSACTION | WORK ] | START TRANSACTION } ...` StartTransaction { modes: Vec }, /// `SET TRANSACTION ...` @@ -1523,31 +1575,34 @@ pub enum Statement { Grant { privileges: Privileges, objects: GrantObjects, - grantees: Vec, + grantees: Vec>, with_grant_option: bool, - granted_by: Option, + granted_by: Option>, }, /// REVOKE privileges ON objects FROM grantees Revoke { privileges: Privileges, objects: GrantObjects, - grantees: Vec, - granted_by: Option, + grantees: Vec>, + granted_by: Option>, cascade: bool, }, /// `DEALLOCATE [ PREPARE ] { name | ALL }` /// /// Note: this is a PostgreSQL-specific statement. - Deallocate { name: Ident, prepare: bool }, + Deallocate { name: Located, prepare: bool }, /// `EXECUTE name [ ( parameter [, ...] ) ]` /// /// Note: this is a PostgreSQL-specific statement. - Execute { name: Ident, parameters: Vec }, + Execute { + name: Located, + parameters: Vec, + }, /// `PREPARE name [ ( data_type [, ...] ) ] AS statement` /// /// Note: this is a PostgreSQL-specific statement. Prepare { - name: Ident, + name: Located, data_types: Vec, statement: Box, }, @@ -1583,7 +1638,7 @@ pub enum Statement { format: Option, }, /// SAVEPOINT -- define a new savepoint within the current transaction - Savepoint { name: Ident }, + Savepoint { name: Located }, // MERGE INTO statement, based on Snowflake. See Merge { // optional INTO keyword @@ -2844,7 +2899,7 @@ pub struct OnConflict { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum ConflictTarget { - Columns(Vec), + Columns(Vec>), OnConstraint(ObjectName), } #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -3026,13 +3081,21 @@ pub enum Action { Create, Delete, Execute, - Insert { columns: Option> }, - References { columns: Option> }, - Select { columns: Option> }, + Insert { + columns: Option>>, + }, + References { + columns: Option>>, + }, + Select { + columns: Option>>, + }, Temporary, Trigger, Truncate, - Update { columns: Option> }, + Update { + columns: Option>>, + }, Usage, } @@ -3119,7 +3182,7 @@ impl fmt::Display for GrantObjects { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Assignment { - pub id: Vec, + pub id: Vec>, pub value: Expr, } @@ -3154,7 +3217,10 @@ impl fmt::Display for FunctionArgExpr { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum FunctionArg { - Named { name: Ident, arg: FunctionArgExpr }, + Named { + name: Located, + arg: FunctionArgExpr, + }, Unnamed(FunctionArgExpr), } @@ -3172,7 +3238,7 @@ impl fmt::Display for FunctionArg { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum CloseCursor { All, - Specific { name: Ident }, + Specific { name: Located }, } impl fmt::Display for CloseCursor { @@ -3438,7 +3504,7 @@ pub enum HiveDistributionStyle { columns: Vec, }, CLUSTERED { - columns: Vec, + columns: Vec>, sorted_by: Vec, num_buckets: i32, }, @@ -3485,7 +3551,7 @@ pub struct HiveFormat { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct SqlOption { - pub name: Ident, + pub name: Located, pub value: Value, } @@ -3650,7 +3716,7 @@ pub enum OnCommit { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum CopyOption { /// FORMAT format_name - Format(Ident), + Format(Located), /// FREEZE \[ boolean \] Freeze(bool), /// DELIMITER 'delimiter_character' @@ -3664,11 +3730,11 @@ pub enum CopyOption { /// ESCAPE 'escape_character' Escape(char), /// FORCE_QUOTE { ( column_name [, ...] ) | * } - ForceQuote(Vec), + ForceQuote(Vec>), /// FORCE_NOT_NULL ( column_name [, ...] ) - ForceNotNull(Vec), + ForceNotNull(Vec>), /// FORCE_NULL ( column_name [, ...] ) - ForceNull(Vec), + ForceNull(Vec>), /// ENCODING 'encoding_name' Encoding(String), } @@ -3739,9 +3805,9 @@ pub enum CopyLegacyCsvOption { /// ESCAPE \[ AS \] 'escape_character' Escape(char), /// FORCE QUOTE { column_name [, ...] | * } - ForceQuote(Vec), + ForceQuote(Vec>), /// FORCE NOT NULL column_name [, ...] - ForceNotNull(Vec), + ForceNotNull(Vec>), } impl fmt::Display for CopyLegacyCsvOption { @@ -3771,7 +3837,7 @@ pub enum MergeClause { MatchedDelete(Option), NotMatched { predicate: Option, - columns: Vec, + columns: Vec>, values: Values, }, } @@ -3914,7 +3980,7 @@ impl fmt::Display for DropFunctionDesc { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct OperateFunctionArg { pub mode: Option, - pub name: Option, + pub name: Option>, pub data_type: DataType, pub default_expr: Option, } @@ -4024,7 +4090,7 @@ impl fmt::Display for FunctionDefinition { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct CreateFunctionBody { /// LANGUAGE lang_name - pub language: Option, + pub language: Option>, /// IMMUTABLE | STABLE | VOLATILE pub behavior: Option, /// AS 'definition' @@ -4088,9 +4154,9 @@ pub enum SchemaName { /// Only schema name specified: ``. Simple(ObjectName), /// Only authorization identifier specified: `AUTHORIZATION `. - UnnamedAuthorization(Ident), + UnnamedAuthorization(Located), /// Both schema name and authorization identifier specified: ` AUTHORIZATION `. - NamedAuthorization(ObjectName, Ident), + NamedAuthorization(ObjectName, Located), } impl fmt::Display for SchemaName { diff --git a/src/ast/query.rs b/src/ast/query.rs index f63234b09..9d237f908 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -20,6 +20,7 @@ use serde::{Deserialize, Serialize}; use sqlparser_derive::{Visit, VisitMut}; use crate::ast::*; +use crate::location::Located; /// The most complete variant of a `SELECT` query expression, optionally /// including `WITH`, `UNION` / other set operations, and `ORDER BY`. @@ -283,7 +284,7 @@ pub struct LateralView { /// LATERAL VIEW table name pub lateral_view_name: ObjectName, /// LATERAL VIEW optional column aliases - pub lateral_col_alias: Vec, + pub lateral_col_alias: Vec>, /// LATERAL VIEW OUTER pub outer: bool, } @@ -337,7 +338,7 @@ impl fmt::Display for With { pub struct Cte { pub alias: TableAlias, pub query: Box, - pub from: Option, + pub from: Option>, } impl fmt::Display for Cte { @@ -358,7 +359,7 @@ pub enum SelectItem { /// Any expression, not followed by `[ AS ] alias` UnnamedExpr(Expr), /// An expression, followed by `[ AS ] alias` - ExprWithAlias { expr: Expr, alias: Ident }, + ExprWithAlias { expr: Expr, alias: Located }, /// `alias.*` or even `schema.table.*` QualifiedWildcard(ObjectName, WildcardAdditionalOptions), /// An unqualified `*` @@ -375,8 +376,8 @@ pub enum SelectItem { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct IdentWithAlias { - pub ident: Ident, - pub alias: Ident, + pub ident: Located, + pub alias: Located, } impl fmt::Display for IdentWithAlias { @@ -430,13 +431,13 @@ pub enum ExcludeSelectItem { /// ```plaintext /// /// ``` - Single(Ident), + Single(Located), /// Multiple column names inside parenthesis. /// # Syntax /// ```plaintext /// (, , ...) /// ``` - Multiple(Vec), + Multiple(Vec>), } impl fmt::Display for ExcludeSelectItem { @@ -506,9 +507,9 @@ impl fmt::Display for RenameSelectItem { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct ExceptSelectItem { /// First guaranteed column. - pub first_element: Ident, + pub first_element: Located, /// Additional columns. This list can be empty. - pub additional_elements: Vec, + pub additional_elements: Vec>, } impl fmt::Display for ExceptSelectItem { @@ -609,7 +610,7 @@ pub enum TableFactor { alias: Option, array_expr: Box, with_offset: bool, - with_offset_alias: Option, + with_offset_alias: Option>, }, /// Represents a parenthesized table factor. The SQL spec only allows a /// join expression (`(foo bar [ baz ... ])`) to be nested, @@ -701,8 +702,8 @@ impl fmt::Display for TableFactor { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct TableAlias { - pub name: Ident, - pub columns: Vec, + pub name: Located, + pub columns: Vec>, } impl fmt::Display for TableAlias { @@ -838,7 +839,7 @@ pub enum JoinOperator { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum JoinConstraint { On(Expr), - Using(Vec), + Using(Vec>), Natural, None, } diff --git a/src/lib.rs b/src/lib.rs index 75209b054..0ae3cd4c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,7 @@ pub mod ast; #[macro_use] pub mod dialect; pub mod keywords; +pub mod location; pub mod parser; pub mod tokenizer; diff --git a/src/location.rs b/src/location.rs new file mode 100644 index 000000000..28ea14804 --- /dev/null +++ b/src/location.rs @@ -0,0 +1,169 @@ +use core::cmp::Eq; +use core::cmp::Ordering; +use core::hash::{Hash, Hasher}; +use std::fmt; + +/// Location in input string +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct Location { + /// Line number, starting from 1 + pub line: u64, + /// Line column, starting from 1 + pub column: u64, +} + +impl Location { + pub fn valid(&self) -> bool { + self.line > 0 && self.column > 0 + } +} + +impl Ord for Location { + fn cmp(&self, other: &Self) -> Ordering { + if self.line == other.line { + self.column.cmp(&other.column) + } else { + self.line.cmp(&other.line) + } + } +} + +impl PartialOrd for Location { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Range { + pub start: Location, + pub end: Location, +} + +impl Range { + pub fn valid(&self) -> bool { + self.start.valid() && self.end.valid() + } + + pub fn into_option(self) -> Option { + if self.valid() { + Some(self) + } else { + None + } + } +} + +pub struct Located> { + value: T, + location: L, +} + +impl Clone for Located +where + T: Clone, + L: Clone, +{ + fn clone(&self) -> Self { + Located { + value: self.value.clone(), + location: self.location.clone(), + } + } +} + +impl fmt::Debug for Located +where + T: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.value, f) + } +} + +impl fmt::Display for Located +where + T: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.value, f) + } +} + +impl Eq for Located +where + T: Eq, + L: Eq, +{ +} + +impl Hash for Located +where + T: Hash, +{ + fn hash(&self, state: &mut H) { + self.value.hash(state); + } +} + +impl PartialEq for Located +where + T: PartialEq, + L: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + self.value.eq(&other.value) && self.location.eq(&other.location) + } +} + +impl Ord for Located +where + T: Ord, + L: Ord, +{ + fn cmp(&self, other: &Self) -> Ordering { + self.value + .cmp(&other.value) + .then_with(|| self.location.cmp(&other.location)) + } +} + +impl PartialOrd for Located +where + T: PartialOrd, + L: PartialOrd, +{ + fn partial_cmp(&self, other: &Self) -> Option { + match self.value.partial_cmp(&other.value) { + Some(Ordering::Equal) => self.location.partial_cmp(&other.location), + Some(o) => Some(o), + None => None, + } + } +} + +impl Located { + pub fn new(value: T, location: L) -> Located { + Located { value, location } + } + + pub fn location(&self) -> &L { + &self.location + } + + pub fn get(&self) -> &T { + &self.value + } + + pub fn into_inner(self) -> T { + self.value + } +} + +impl std::ops::Deref for Located { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.value + } +} diff --git a/src/parser.rs b/src/parser.rs index 34398ec4c..f6d8de16c 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -31,6 +31,7 @@ use crate::ast::helpers::stmt_create_table::CreateTableBuilder; use crate::ast::*; use crate::dialect::*; use crate::keywords::{self, Keyword}; +use crate::location::*; use crate::tokenizer::*; #[derive(Debug, Clone, PartialEq, Eq)] @@ -198,7 +199,9 @@ const DEFAULT_REMAINING_DEPTH: usize = 50; pub struct Parser<'a> { tokens: Vec, /// The index of the first unprocessed token in `self.tokens` - index: usize, + pub index: usize, + eof_loc: Location, + autocomplete: Vec>, /// The current dialect to use dialect: &'a dyn Dialect, /// ensure the stack does not overflow by limiting recusion depth @@ -222,9 +225,38 @@ impl<'a> Parser<'a> { /// # } /// ``` pub fn new(dialect: &'a dyn Dialect) -> Self { - Self { - tokens: vec![], + Parser::new_without_locations(vec![], dialect) + } + + pub fn new_without_locations(tokens: Vec, dialect: &'a dyn Dialect) -> Self { + // Put in dummy locations + let tokens_with_locations: Vec = tokens + .into_iter() + .map(|token| TokenWithLocation { + token, + location: Location { line: 0, column: 0 }, + }) + .collect(); + Parser::new_with_locations( + tokens_with_locations, + Location { line: 0, column: 0 }, + dialect, + ) + } + + /// Parse the specified tokens + pub fn new_with_locations( + tokens: Vec, + eof_loc: Location, + dialect: &'a dyn Dialect, + ) -> Self { + let mut autocomplete: Vec<_> = tokens.iter().map(|_| Vec::new()).collect(); + autocomplete.push(Vec::new()); + Parser { + tokens, index: 0, + eof_loc, + autocomplete, dialect, recursion_counter: RecursionCounter::new(DEFAULT_REMAINING_DEPTH), } @@ -312,7 +344,7 @@ impl<'a> Parser<'a> { expecting_statement_delimiter = false; } - if self.peek_token() == Token::EOF { + if self.peek_token().token == Token::EOF { break; } if expecting_statement_delimiter { @@ -355,6 +387,46 @@ impl<'a> Parser<'a> { return statement; } + self.autocomplete_tokens(&[Token::LParen]); + self.autocomplete_keywords(&[ + Keyword::KILL, + Keyword::DESCRIBE, + Keyword::EXPLAIN, + Keyword::ANALYZE, + Keyword::SELECT, + Keyword::WITH, + Keyword::VALUES, + Keyword::TRUNCATE, + Keyword::MSCK, + Keyword::CREATE, + Keyword::CACHE, + Keyword::DROP, + Keyword::DISCARD, + Keyword::DECLARE, + Keyword::FETCH, + Keyword::DELETE, + Keyword::INSERT, + Keyword::UNCACHE, + Keyword::UPDATE, + Keyword::ALTER, + Keyword::COPY, + Keyword::CLOSE, + Keyword::SET, + Keyword::SHOW, + Keyword::USE, + Keyword::GRANT, + Keyword::REVOKE, + Keyword::START, + Keyword::BEGIN, + Keyword::SAVEPOINT, + Keyword::COMMIT, + Keyword::ROLLBACK, + Keyword::ASSERT, + Keyword::DEALLOCATE, + Keyword::EXECUTE, + Keyword::PREPARE, + Keyword::MERGE, + ]); let next_token = self.next_token(); match &next_token.token { Token::Word(w) => match w.keyword { @@ -513,15 +585,18 @@ impl<'a> Parser<'a> { pub fn parse_wildcard_expr(&mut self) -> Result { let index = self.index; + self.autocomplete_tokens(&[Token::Mul, Token::make_ident("")]); + let range = self.peek_token_range(); let next_token = self.next_token(); match next_token.token { Token::Word(w) if self.peek_token().token == Token::Period => { - let mut id_parts: Vec = vec![w.to_ident()]; + let mut id_parts: Vec> = vec![w.to_located_ident(range)]; while self.consume_token(&Token::Period) { + let range = self.peek_token_range(); let next_token = self.next_token(); match next_token.token { - Token::Word(w) => id_parts.push(w.to_ident()), + Token::Word(w) => id_parts.push(w.to_located_ident(range)), Token::Mul => { return Ok(WildcardExpr::QualifiedWildcard(ObjectName(id_parts))); } @@ -652,6 +727,49 @@ impl<'a> Parser<'a> { } })); + self.autocomplete_tokens(&[ + Token::LBracket, + Token::Minus, + Token::Plus, + Token::DoubleExclamationMark, + Token::PGSquareRoot, + Token::PGCubeRoot, + Token::AtSign, + Token::Tilde, + Token::LParen, + Token::Colon, + ]); + self.autocomplete_keywords(&[ + Keyword::TRUE, + Keyword::FALSE, + Keyword::NULL, + Keyword::CURRENT_CATALOG, + Keyword::CURRENT_USER, + Keyword::SESSION_USER, + Keyword::USER, + Keyword::CURRENT_TIMESTAMP, + Keyword::CURRENT_TIME, + Keyword::CURRENT_DATE, + Keyword::LOCALTIME, + Keyword::LOCALTIMESTAMP, + Keyword::CASE, + Keyword::CAST, + Keyword::TRY_CAST, + Keyword::SAFE_CAST, + Keyword::EXISTS, + Keyword::EXTRACT, + Keyword::CEIL, + Keyword::FLOOR, + Keyword::POSITION, + Keyword::SUBSTRING, + Keyword::OVERLAY, + Keyword::TRIM, + Keyword::INTERVAL, + Keyword::LISTAGG, + Keyword::ARRAY, + Keyword::NOT, + ]); + let range = self.peek_token_range(); let next_token = self.next_token(); let expr = match next_token.token { Token::Word(w) => match w.keyword { @@ -666,7 +784,7 @@ impl<'a> Parser<'a> { if dialect_of!(self is PostgreSqlDialect | GenericDialect) => { Ok(Expr::Function(Function { - name: ObjectName(vec![w.to_ident()]), + name: ObjectName(vec![w.to_located_ident(range)]), args: vec![], over: None, distinct: false, @@ -678,7 +796,7 @@ impl<'a> Parser<'a> { | Keyword::CURRENT_DATE | Keyword::LOCALTIME | Keyword::LOCALTIMESTAMP => { - self.parse_time_functions(ObjectName(vec![w.to_ident()])) + self.parse_time_functions(ObjectName(vec![w.to_located_ident(range)])) } Keyword::CASE => self.parse_case_expr(), Keyword::CAST => self.parse_cast_expr(), @@ -695,12 +813,12 @@ impl<'a> Parser<'a> { Keyword::INTERVAL => self.parse_interval(), Keyword::LISTAGG => self.parse_listagg_expr(), // Treat ARRAY[1,2,3] as an array [1,2,3], otherwise try as subquery or a function call - Keyword::ARRAY if self.peek_token() == Token::LBracket => { + Keyword::ARRAY if self.peek_token().token == Token::LBracket => { self.expect_token(&Token::LBracket)?; self.parse_array_expr(true) } Keyword::ARRAY - if self.peek_token() == Token::LParen + if self.peek_token().token == Token::LParen && !dialect_of!(self is ClickHouseDialect) => { self.expect_token(&Token::LParen)?; @@ -713,29 +831,38 @@ impl<'a> Parser<'a> { } // Here `w` is a word, check if it's a part of a multi-part // identifier, a function call, or a simple identifier: - _ => match self.peek_token().token { - Token::LParen | Token::Period => { - let mut id_parts: Vec = vec![w.to_ident()]; - while self.consume_token(&Token::Period) { - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => id_parts.push(w.to_ident()), - _ => { - return self - .expected("an identifier or a '*' after '.'", next_token); + _ => { + self.autocomplete_tokens(&[Token::LParen, Token::Period]); + match self.peek_token().token { + Token::LParen | Token::Period => { + let mut id_parts: Vec> = vec![w.to_located_ident(range)]; + while self.consume_token(&Token::Period) { + self.autocomplete_tokens(&[Token::make_ident( + ObjectName(id_parts.clone()).to_string().as_str(), + )]); + let range = self.peek_token_range(); + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => id_parts.push(w.to_located_ident(range)), + _ => { + return self.expected( + "an identifier or a '*' after '.'", + next_token, + ); + } } } - } - if self.consume_token(&Token::LParen) { - self.prev_token(); - self.parse_function(ObjectName(id_parts)) - } else { - Ok(Expr::CompoundIdentifier(id_parts)) + if self.consume_token(&Token::LParen) { + self.prev_token(); + self.parse_function(ObjectName(id_parts)) + } else { + Ok(Expr::CompoundIdentifier(id_parts)) + } } + _ => Ok(Expr::Identifier(w.to_located_ident(range))), } - _ => Ok(Expr::Identifier(w.to_ident())), - }, + } }, // End of Token::Word // array `[1, 2, 3]` Token::LBracket => self.parse_array_expr(false), @@ -802,9 +929,11 @@ impl<'a> Parser<'a> { if !self.consume_token(&Token::Period) { Ok(expr) } else { + self.autocomplete_tokens(&[Token::make_ident(expr.to_string().as_str())]); + let range = self.peek_token_range(); let tok = self.next_token(); let key = match tok.token { - Token::Word(word) => word.to_ident(), + Token::Word(word) => word.to_located_ident(range), _ => return parser_err!(format!("Expected identifier, found: {}", tok)), }; Ok(Expr::CompositeAccess { @@ -889,6 +1018,7 @@ impl<'a> Parser<'a> { } pub fn parse_window_frame_units(&mut self) -> Result { + self.autocomplete_keywords(&[Keyword::ROWS, Keyword::RANGE, Keyword::GROUPS]); let next_token = self.next_token(); match &next_token.token { Token::Word(w) => match w.keyword { @@ -1217,6 +1347,7 @@ impl<'a> Parser<'a> { } pub fn parse_trim_where(&mut self) -> Result { + self.autocomplete_keywords(&[Keyword::BOTH, Keyword::LEADING, Keyword::TRAILING]); let next_token = self.next_token(); match &next_token.token { Token::Word(w) => match w.keyword { @@ -1232,6 +1363,7 @@ impl<'a> Parser<'a> { /// Parses an array expression `[ex1, ex2, ..]` /// if `named` is `true`, came from an expression like `ARRAY[ex1, ex2]` pub fn parse_array_expr(&mut self, named: bool) -> Result { + self.autocomplete_tokens(&[Token::RBracket]); if self.peek_token().token == Token::RBracket { let _ = self.next_token(); // consume ] Ok(Expr::Array(Array { @@ -1368,6 +1500,33 @@ impl<'a> Parser<'a> { // EXTRACT supports a wider set of date/time fields than interval qualifiers, // so this function may need to be split in two. pub fn parse_date_time_field(&mut self) -> Result { + self.autocomplete_keywords(&[ + Keyword::YEAR, + Keyword::MONTH, + Keyword::WEEK, + Keyword::DAY, + Keyword::HOUR, + Keyword::MINUTE, + Keyword::SECOND, + Keyword::CENTURY, + Keyword::DECADE, + Keyword::DOY, + Keyword::DOW, + Keyword::EPOCH, + Keyword::ISODOW, + Keyword::ISOYEAR, + Keyword::JULIAN, + Keyword::MICROSECOND, + Keyword::MICROSECONDS, + Keyword::MILLENIUM, + Keyword::MILLENNIUM, + Keyword::MILLISECOND, + Keyword::MILLISECONDS, + Keyword::QUARTER, + Keyword::TIMEZONE, + Keyword::TIMEZONE_HOUR, + Keyword::TIMEZONE_MINUTE, + ]); let next_token = self.next_token(); match &next_token.token { Token::Word(w) => match w.keyword { @@ -1406,6 +1565,7 @@ impl<'a> Parser<'a> { } pub fn parse_not(&mut self) -> Result { + self.autocomplete_keywords(&[Keyword::EXISTS]); match self.peek_token().token { Token::Word(w) => match w.keyword { Keyword::EXISTS => { @@ -1504,6 +1664,33 @@ impl<'a> Parser<'a> { // // Note that PostgreSQL allows omitting the qualifier, so we provide // this more general implementation. + self.autocomplete_keywords(&[ + Keyword::YEAR, + Keyword::MONTH, + Keyword::WEEK, + Keyword::DAY, + Keyword::HOUR, + Keyword::MINUTE, + Keyword::SECOND, + Keyword::CENTURY, + Keyword::DECADE, + Keyword::DOW, + Keyword::DOY, + Keyword::EPOCH, + Keyword::ISODOW, + Keyword::ISOYEAR, + Keyword::JULIAN, + Keyword::MICROSECOND, + Keyword::MICROSECONDS, + Keyword::MILLENIUM, + Keyword::MILLENNIUM, + Keyword::MILLISECOND, + Keyword::MILLISECONDS, + Keyword::QUARTER, + Keyword::TIMEZONE, + Keyword::TIMEZONE_HOUR, + Keyword::TIMEZONE_MINUTE, + ]); let leading_field = match self.peek_token().token { Token::Word(kw) if [ @@ -1584,6 +1771,50 @@ impl<'a> Parser<'a> { return infix; } + self.autocomplete_tokens(&[ + Token::Spaceship, + Token::DoubleEq, + Token::Eq, + Token::Neq, + Token::Gt, + Token::GtEq, + Token::Lt, + Token::LtEq, + Token::Plus, + Token::Minus, + Token::Mul, + Token::Mod, + Token::StringConcat, + Token::Pipe, + Token::Caret, + Token::Ampersand, + Token::Div, + Token::ShiftLeft, + Token::ShiftRight, + Token::Sharp, + Token::Tilde, + Token::TildeAsterisk, + Token::ExclamationMarkTilde, + Token::ExclamationMarkTildeAsterisk, + Token::DoubleColon, + Token::ExclamationMark, + Token::LBracket, + Token::Arrow, + Token::LongArrow, + Token::HashArrow, + Token::HashLongArrow, + ]); + self.autocomplete_keywords(&[ + Keyword::IS, + Keyword::AT, + Keyword::NOT, + Keyword::IN, + Keyword::BETWEEN, + Keyword::LIKE, + Keyword::ILIKE, + Keyword::SIMILAR, + ]); + let tok = self.next_token(); let regular_binary_operator = match &tok.token { @@ -1761,15 +1992,15 @@ impl<'a> Parser<'a> { // Can only happen if `get_next_precedence` got out of sync with this function _ => parser_err!(format!("No infix parser for token {:?}", tok.token)), } - } else if Token::DoubleColon == tok { + } else if Token::DoubleColon == tok.token { self.parse_pg_cast(expr) - } else if Token::ExclamationMark == tok { + } else if Token::ExclamationMark == tok.token { // PostgreSQL factorial operation Ok(Expr::UnaryOp { op: UnaryOperator::PGPostfixFactorial, expr: Box::new(expr), }) - } else if Token::LBracket == tok { + } else if Token::LBracket == tok.token { if dialect_of!(self is PostgreSqlDialect | GenericDialect) { // parse index return self.parse_array_index(expr); @@ -1934,9 +2165,9 @@ impl<'a> Parser<'a> { let token = self.peek_token(); debug!("get_next_precedence() {:?}", token); - let token_0 = self.peek_nth_token(0); - let token_1 = self.peek_nth_token(1); - let token_2 = self.peek_nth_token(2); + let token_0 = self.peek_nth_token(0).1; + let token_1 = self.peek_nth_token(1).1; + let token_2 = self.peek_nth_token(2).1; debug!("0: {token_0} 1: {token_1} 2: {token_2}"); match token.token { Token::Word(w) if w.keyword == Keyword::OR => Ok(Self::OR_PREC), @@ -1944,7 +2175,10 @@ impl<'a> Parser<'a> { Token::Word(w) if w.keyword == Keyword::XOR => Ok(Self::XOR_PREC), Token::Word(w) if w.keyword == Keyword::AT => { - match (self.peek_nth_token(1).token, self.peek_nth_token(2).token) { + match ( + self.peek_nth_token(1).1.token, + self.peek_nth_token(2).1.token, + ) { (Token::Word(w), Token::Word(w2)) if w.keyword == Keyword::TIME && w2.keyword == Keyword::ZONE => { @@ -1954,7 +2188,7 @@ impl<'a> Parser<'a> { } } - Token::Word(w) if w.keyword == Keyword::NOT => match self.peek_nth_token(1).token { + Token::Word(w) if w.keyword == Keyword::NOT => match self.peek_nth_token(1).1.token { // The precedence of NOT varies depending on keyword that // follows it. If it is followed by IN, BETWEEN, or LIKE, // it takes on the precedence of those tokens. Otherwise it @@ -2008,14 +2242,21 @@ impl<'a> Parser<'a> { } } + pub fn eof_token(&self) -> TokenWithLocation { + TokenWithLocation { + token: Token::EOF, + location: self.eof_loc.clone(), + } + } + /// Return the first non-whitespace token that has not yet been processed /// (or None if reached end-of-file) pub fn peek_token(&self) -> TokenWithLocation { - self.peek_nth_token(0) + self.peek_nth_token(0).1 } /// Return nth non-whitespace token that has not yet been processed - pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation { + pub fn peek_nth_token(&self, mut n: usize) -> (usize, TokenWithLocation) { let mut index = self.index; loop { index += 1; @@ -2026,10 +2267,10 @@ impl<'a> Parser<'a> { }) => continue, non_whitespace => { if n == 0 { - return non_whitespace.cloned().unwrap_or(TokenWithLocation { - token: Token::EOF, - location: Location { line: 0, column: 0 }, - }); + return ( + index - 1, + non_whitespace.cloned().unwrap_or(self.eof_token()), + ); } n -= 1; } @@ -2048,11 +2289,7 @@ impl<'a> Parser<'a> { token: Token::Whitespace(_), location: _, }) => continue, - token => { - return token - .cloned() - .unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF)) - } + token => return token.cloned().unwrap_or(self.eof_token()), } } } @@ -2089,6 +2326,7 @@ impl<'a> Parser<'a> { /// Look for an expected keyword and consume it if it exists #[must_use] pub fn parse_keyword(&mut self, expected: Keyword) -> bool { + self.autocomplete_keywords(&[expected.clone()]); match self.peek_token().token { Token::Word(w) if expected == w.keyword => { self.next_token(); @@ -2116,6 +2354,7 @@ impl<'a> Parser<'a> { /// Look for one of the given keywords and return the one that matches. #[must_use] pub fn parse_one_of_keywords(&mut self, keywords: &[Keyword]) -> Option { + self.autocomplete_keywords(keywords); match self.peek_token().token { Token::Word(w) => { keywords @@ -2164,7 +2403,8 @@ impl<'a> Parser<'a> { /// Consume the next token if it matches the expected token, otherwise return false #[must_use] pub fn consume_token(&mut self, expected: &Token) -> bool { - if self.peek_token() == *expected { + self.autocomplete_tokens(&[expected.clone()]); + if self.peek_token().token == *expected { self.next_token(); true } else { @@ -2223,6 +2463,168 @@ impl<'a> Parser<'a> { Ok(values) } + pub fn current_index(&self) -> usize { + self.index + } + + pub fn reset(&mut self, index: usize) { + self.index = index; + } + + pub fn start_location(&self, index: usize) -> Location { + self.tokens + .get(index) + .unwrap_or(&self.eof_token()) + .location + .clone() + } + + pub fn end_location(&self, index: usize) -> Location { + if !self.has_locations() { + return Location { line: 0, column: 0 }; + } + + let mut end = self + .tokens + .get(index + 1) + .unwrap_or(&self.eof_token()) + .location + .clone(); + + if end.column > 1 { + end.column -= 1; + } else { + // The previous token must be a newline, so we simply mark that as the end. + end = self + .tokens + .get(index) + .unwrap_or(&self.eof_token()) + .location + .clone(); + } + assert!(end.valid()); + + end + } + + pub fn has_locations(&self) -> bool { + self.eof_loc.valid() + } + + pub fn peek_start_location(&self) -> Location { + self.peek_token().location + } + + pub fn peek_end_location(&self) -> Location { + self.end_location(self.peek_nth_token(0).0) + } + + pub fn prev_end_location(&self) -> Location { + self.end_location(self.index - 1) + } + + fn peek_token_range(&self) -> Range { + Range { + start: self.peek_start_location(), + end: self.peek_end_location(), + } + } + + fn track_autocomplete(&mut self, token: Token) { + let index = std::cmp::min(self.index, self.autocomplete.len()); + if index > 0 { + self.autocomplete[index - 1].push(token); + } + } + + pub fn autocomplete_tokens(&mut self, tokens: &[Token]) { + let index = self.index; + self.next_token(); + for token in tokens { + self.track_autocomplete(token.clone()); + } + self.index = index; + } + + pub fn autocomplete_keywords(&mut self, keywords: &[Keyword]) { + let index = self.index; + self.next_token(); + for keyword in keywords { + self.track_autocomplete(Token::from_keyword(keyword.clone())); + } + self.index = index; + } + + pub fn get_autocomplete(&self, loc: Location) -> (TokenWithLocation, Vec) { + let eof_autocomplete = self.autocomplete.last().unwrap().clone(); + if self.tokens.len() == 0 || self.tokens[0].location > loc { + return (self.eof_token(), eof_autocomplete); + } + + // Find the index of the token this location is inside + // + let mut index = self.tokens.partition_point(|a| { + if a.location.line == loc.line { + a.location.column <= loc.column + } else { + a.location.line < loc.line + } + }) - 1; + + // If we're at the start of our token, the previous token is a word and the current token + // is whitespace, then return the suggestions for the previous token. + // + if index > 0 + && self.tokens[index].location == loc + && matches!(self.tokens[index - 1].token, Token::Word(_)) + && matches!(self.tokens[index].token, Token::Whitespace(_)) + { + return ( + self.tokens[index - 1].clone(), + self.autocomplete[index - 1].clone(), + ); + } + + // If the current token is not a word, skip its suggestions and look for subsequent ones. + // + if !matches!(self.tokens[index].token, Token::Word(_)) { + index += 1; + } + + // Find the next token that has suggestions and return those. + // + for i in index..self.tokens.len() { + if self.autocomplete[i].len() > 0 { + let token = self.tokens[i].clone(); + return (token, self.autocomplete[i].clone()); + } + } + + // If we've reached the end of the stream without finding any suggestions, return the EOF + // suggestions. As a special case, do not return any suggestions if the last token was a + // word, since it has no whitespace after it. + // + if match self.tokens.last().unwrap().token { + Token::Word(_) + | Token::Number { .. } + | Token::SingleQuotedString(_) + | Token::DoubleQuotedString(_) + | Token::NationalStringLiteral(_) + | Token::EscapedStringLiteral(_) + | Token::HexStringLiteral(_) + | Token::Placeholder(_) => true, + _ => false, + } { + return (self.eof_token(), Vec::new()); + } else { + return (self.eof_token(), eof_autocomplete); + } + } + + pub fn get_autocompletes(&self) -> Vec> { + self.autocomplete.clone() + } + /// Run a parser method `f`, reverting back to the current position /// if unsuccessful. #[must_use] @@ -2337,14 +2739,14 @@ impl<'a> Parser<'a> { table_flag = Some(self.parse_object_name()?); if self.parse_keyword(Keyword::TABLE) { let table_name = self.parse_object_name()?; - if self.peek_token() != Token::EOF { + if self.peek_token().token != Token::EOF { if let Token::Word(word) = self.peek_token().token { if word.keyword == Keyword::OPTIONS { options = self.parse_options(Keyword::OPTIONS)? } }; - if self.peek_token() != Token::EOF { + if self.peek_token().token != Token::EOF { let (a, q) = self.parse_as_query()?; has_as = a; query = Some(q); @@ -2367,7 +2769,7 @@ impl<'a> Parser<'a> { }) } } else { - if self.peek_token() == Token::EOF { + if self.peek_token().token == Token::EOF { self.prev_token(); } self.expected("a `TABLE` keyword", self.peek_token()) @@ -2377,6 +2779,7 @@ impl<'a> Parser<'a> { /// Parse 'AS' before as query,such as `WITH XXX AS SELECT XXX` oer `CACHE TABLE AS SELECT XXX` pub fn parse_as_query(&mut self) -> Result<(bool, Query), ParserError> { + self.autocomplete_keywords(&[Keyword::AS]); match self.peek_token().token { Token::Word(word) => match word.keyword { Keyword::AS => { @@ -2663,6 +3066,15 @@ impl<'a> Parser<'a> { } pub fn parse_file_format(&mut self) -> Result { + self.autocomplete_keywords(&[ + Keyword::AVRO, + Keyword::JSONFILE, + Keyword::ORC, + Keyword::PARQUET, + Keyword::RCFILE, + Keyword::SEQUENCEFILE, + Keyword::TEXTFILE, + ]); let next_token = self.next_token(); match &next_token.token { Token::Word(w) => match w.keyword { @@ -2680,6 +3092,7 @@ impl<'a> Parser<'a> { } pub fn parse_analyze_format(&mut self) -> Result { + self.autocomplete_keywords(&[Keyword::TEXT, Keyword::GRAPHVIZ, Keyword::JSON]); let next_token = self.next_token(); match &next_token.token { Token::Word(w) => match w.keyword { @@ -4072,6 +4485,8 @@ impl<'a> Parser<'a> { /// Parse a literal value (numbers, strings, date/time, booleans) pub fn parse_value(&mut self) -> Result { + self.autocomplete_tokens(&[Token::Colon, Token::AtSign]); + self.autocomplete_keywords(&[Keyword::TRUE, Keyword::FALSE, Keyword::NULL]); let next_token = self.next_token(); let location = next_token.location; match next_token.token { @@ -4181,11 +4596,12 @@ impl<'a> Parser<'a> { /// Parse a map key string pub fn parse_map_key(&mut self) -> Result { + let range = self.peek_token_range(); let next_token = self.next_token(); match next_token.token { Token::Word(Word { value, keyword, .. }) if keyword == Keyword::NoKeyword => { - if self.peek_token() == Token::LParen { - return self.parse_function(ObjectName(vec![Ident::new(value)])); + if self.peek_token().token == Token::LParen { + return self.parse_function(ObjectName(vec![Ident::new_located(value, range)])); } Ok(Expr::Value(Value::SingleQuotedString(value))) } @@ -4200,6 +4616,43 @@ impl<'a> Parser<'a> { /// Parse a SQL datatype (in the context of a CREATE TABLE statement for example) pub fn parse_data_type(&mut self) -> Result { + self.autocomplete_keywords(&[ + Keyword::BOOLEAN, + Keyword::FLOAT, + Keyword::REAL, + Keyword::TINYINT, + Keyword::SMALLINT, + Keyword::MEDIUMINT, + Keyword::INT, + Keyword::INTEGER, + Keyword::BIGINT, + Keyword::VARCHAR, + Keyword::NVARCHAR, + Keyword::CHARACTER, + Keyword::CHAR, + Keyword::CLOB, + Keyword::BINARY, + Keyword::VARBINARY, + Keyword::BLOB, + Keyword::UUID, + Keyword::DATE, + Keyword::DATETIME, + Keyword::TIMESTAMP, + Keyword::TIMESTAMPTZ, + Keyword::TIME, + Keyword::TIMETZ, + Keyword::INTERVAL, + Keyword::REGCLASS, + Keyword::STRING, + Keyword::TEXT, + Keyword::BYTEA, + Keyword::NUMERIC, + Keyword::DECIMAL, + Keyword::DEC, + Keyword::ENUM, + Keyword::SET, + Keyword::ARRAY, + ]); let next_token = self.next_token(); let mut data = match next_token.token { Token::Word(w) => match w.keyword { @@ -4391,6 +4844,7 @@ impl<'a> Parser<'a> { Token::SingleQuotedString(value) => values.push(value), _ => self.expected("a string", next_token)?, } + self.autocomplete_tokens(&[Token::Comma, Token::RParen]); let next_token = self.next_token(); match next_token.token { Token::Comma => (), @@ -4415,17 +4869,19 @@ impl<'a> Parser<'a> { pub fn parse_optional_alias( &mut self, reserved_kwds: &[Keyword], - ) -> Result, ParserError> { + ) -> Result>, ParserError> { let after_as = self.parse_keyword(Keyword::AS); + self.autocomplete_tokens(&[Token::make_ident("")]); + let range = self.peek_token_range(); let next_token = self.next_token(); - match next_token.token { + match &next_token.token { // Accept any identifier after `AS` (though many dialects have restrictions on // keywords that may appear here). If there's no `AS`: don't parse keywords, // which may start a construct allowed in this position, to be parsed as aliases. // (For example, in `FROM t1 JOIN` the `JOIN` will always be parsed as a keyword, // not an alias.) Token::Word(w) if after_as || !reserved_kwds.contains(&w.keyword) => { - Ok(Some(w.to_ident())) + Ok(Some(w.to_located_ident(range))) } // MSSQL supports single-quoted strings as aliases for columns // We accept them as table aliases too, although MSSQL does not. @@ -4439,9 +4895,9 @@ impl<'a> Parser<'a> { // character. When it sees such a , your DBMS will // ignore the and treat the multiple strings as // a single ." - Token::SingleQuotedString(s) => Ok(Some(Ident::with_quote('\'', s))), + Token::SingleQuotedString(s) => Ok(Some(Ident::with_quote_located('\'', s, range))), // Support for MySql dialect double quoted string, `AS "HOUR"` for example - Token::DoubleQuotedString(s) => Ok(Some(Ident::with_quote('\"', s))), + Token::DoubleQuotedString(s) => Ok(Some(Ident::with_quote_located('\"', s, range))), _ => { if after_as { return self.expected("an identifier after AS", next_token); @@ -4474,6 +4930,9 @@ impl<'a> Parser<'a> { pub fn parse_object_name(&mut self) -> Result { let mut idents = vec![]; loop { + self.autocomplete_tokens(&[Token::make_ident( + ObjectName(idents.clone()).to_string().as_str(), + )]); idents.push(self.parse_identifier()?); if !self.consume_token(&Token::Period) { break; @@ -4483,12 +4942,13 @@ impl<'a> Parser<'a> { } /// Parse identifiers - pub fn parse_identifiers(&mut self) -> Result, ParserError> { + pub fn parse_identifiers(&mut self) -> Result>, ParserError> { let mut idents = vec![]; loop { + self.autocomplete_tokens(&[Token::make_ident("")]); match self.peek_token().token { Token::Word(w) => { - idents.push(w.to_ident()); + idents.push(w.to_located_ident(self.peek_token_range())); } Token::EOF | Token::Eq => break, _ => {} @@ -4499,12 +4959,14 @@ impl<'a> Parser<'a> { } /// Parse a simple one-word identifier (possibly quoted, possibly a keyword) - pub fn parse_identifier(&mut self) -> Result { + pub fn parse_identifier(&mut self) -> Result, ParserError> { + self.autocomplete_tokens(&[Token::make_ident("")]); + let range = self.peek_token_range(); let next_token = self.next_token(); match next_token.token { - Token::Word(w) => Ok(w.to_ident()), - Token::SingleQuotedString(s) => Ok(Ident::with_quote('\'', s)), - Token::DoubleQuotedString(s) => Ok(Ident::with_quote('\"', s)), + Token::Word(w) => Ok(w.to_located_ident(range)), + Token::SingleQuotedString(s) => Ok(Ident::with_quote_located('\'', s, range)), + Token::DoubleQuotedString(s) => Ok(Ident::with_quote_located('\"', s, range)), _ => self.expected("identifier", next_token), } } @@ -4514,7 +4976,7 @@ impl<'a> Parser<'a> { &mut self, optional: IsOptional, allow_empty: bool, - ) -> Result, ParserError> { + ) -> Result>, ParserError> { if self.consume_token(&Token::LParen) { if allow_empty && self.peek_token().token == Token::RParen { self.next_token(); @@ -4874,6 +5336,7 @@ impl<'a> Parser<'a> { }; loop { + self.autocomplete_keywords(&[Keyword::UNION, Keyword::EXCEPT, Keyword::INTERSECT]); // The query can be optionally followed by a set operator: let op = self.parse_set_operator(&self.peek_token().token); let next_precedence = match op { @@ -5132,8 +5595,10 @@ impl<'a> Parser<'a> { }); } + let start = self.peek_start_location(); let variable = if self.parse_keywords(&[Keyword::TIME, Keyword::ZONE]) { - ObjectName(vec!["TIMEZONE".into()]) + let end = self.peek_end_location(); + ObjectName(vec![Ident::new_located("TIMEZONE", Range { start, end })]) } else { self.parse_object_name()? }; @@ -5376,6 +5841,14 @@ impl<'a> Parser<'a> { } } else { let natural = self.parse_keyword(Keyword::NATURAL); + self.autocomplete_keywords(&[ + Keyword::INNER, + Keyword::JOIN, + Keyword::LEFT, + Keyword::RIGHT, + Keyword::FULL, + ]); + let peek_keyword = if let Token::Word(w) = self.peek_token().token { w.keyword } else { @@ -5768,7 +6241,9 @@ impl<'a> Parser<'a> { Ok((privileges, objects)) } - pub fn parse_grant_permission(&mut self) -> Result<(Keyword, Option>), ParserError> { + pub fn parse_grant_permission( + &mut self, + ) -> Result<(Keyword, Option>>), ParserError> { if let Some(kw) = self.parse_one_of_keywords(&[ Keyword::CONNECT, Keyword::CREATE, @@ -5995,7 +6470,7 @@ impl<'a> Parser<'a> { } pub fn parse_function_args(&mut self) -> Result { - if self.peek_nth_token(1) == Token::RArrow { + if self.peek_nth_token(1).1.token == Token::RArrow { let name = self.parse_identifier()?; self.expect_token(&Token::RArrow)?; @@ -6412,7 +6887,8 @@ impl<'a> Parser<'a> { pub fn parse_merge_clauses(&mut self) -> Result, ParserError> { let mut clauses: Vec = vec![]; loop { - if self.peek_token() == Token::EOF || self.peek_token() == Token::SemiColon { + if self.peek_token().token == Token::EOF || self.peek_token().token == Token::SemiColon + { break; } self.expect_keyword(Keyword::WHEN)?; @@ -6623,6 +7099,10 @@ impl Word { quote_style: self.quote_style, } } + + pub fn to_located_ident(&self, range: Range) -> Located { + self.to_ident().into_located(range) + } } #[cfg(test)] @@ -6635,19 +7115,19 @@ mod tests { fn test_prev_index() { let sql = "SELECT version"; all_dialects().run_parser_method(sql, |parser| { - assert_eq!(parser.peek_token(), Token::make_keyword("SELECT")); - assert_eq!(parser.next_token(), Token::make_keyword("SELECT")); + assert_eq!(parser.peek_token().token, Token::make_keyword("SELECT")); + assert_eq!(parser.next_token().token, Token::make_keyword("SELECT")); parser.prev_token(); - assert_eq!(parser.next_token(), Token::make_keyword("SELECT")); - assert_eq!(parser.next_token(), Token::make_word("version", None)); + assert_eq!(parser.next_token().token, Token::make_keyword("SELECT")); + assert_eq!(parser.next_token().token, Token::make_word("version", None)); parser.prev_token(); - assert_eq!(parser.peek_token(), Token::make_word("version", None)); - assert_eq!(parser.next_token(), Token::make_word("version", None)); - assert_eq!(parser.peek_token(), Token::EOF); + assert_eq!(parser.peek_token().token, Token::make_word("version", None)); + assert_eq!(parser.next_token().token, Token::make_word("version", None)); + assert_eq!(parser.peek_token().token, Token::EOF); parser.prev_token(); - assert_eq!(parser.next_token(), Token::make_word("version", None)); - assert_eq!(parser.next_token(), Token::EOF); - assert_eq!(parser.next_token(), Token::EOF); + assert_eq!(parser.next_token().token, Token::make_word("version", None)); + assert_eq!(parser.next_token().token, Token::EOF); + assert_eq!(parser.next_token().token, Token::EOF); parser.prev_token(); }); } @@ -6686,6 +7166,17 @@ mod tests { }); } + #[test] + fn test_end_location() { + let sql = "SELECT 1\nSELECT a"; + let mut tokenizer = Tokenizer::new(&GenericDialect {}, sql); + let (tokens, eof_loc) = tokenizer.tokenize_with_location().unwrap(); + + let mut parser = Parser::new_with_locations(tokens, eof_loc, &GenericDialect {}); + parser.parse_query().unwrap(); + assert_eq!(parser.prev_end_location(), Location { line: 1, column: 9 }); + } + #[cfg(test)] mod test_parse_data_type { use crate::ast::{ diff --git a/src/tokenizer.rs b/src/tokenizer.rs index dcf128542..5524d13f2 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -38,6 +38,7 @@ use crate::ast::DollarQuotedString; use crate::dialect::SnowflakeDialect; use crate::dialect::{Dialect, MySqlDialect}; use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; +pub use crate::location::Location; /// SQL Token enumeration #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -264,6 +265,19 @@ impl Token { }, }) } + + pub fn make_ident(ident: &str) -> Self { + Token::make_word(ident, Some('\"')) + } + + pub fn from_keyword(keyword: Keyword) -> Self { + let index = ALL_KEYWORDS_INDEX.binary_search(&keyword).unwrap(); + Token::Word(Word { + value: ALL_KEYWORDS[index].to_string(), + quote_style: None, + keyword, + }) + } } /// A keyword (like SELECT) or an optionally quoted SQL identifier @@ -329,15 +343,6 @@ impl fmt::Display for Whitespace { } } -/// Location in input string -#[derive(Debug, Eq, PartialEq, Clone)] -pub struct Location { - /// Line number, starting from 1 - pub line: u64, - /// Line column, starting from 1 - pub column: u64, -} - /// A [Token] with [Location] attached to it #[derive(Debug, Eq, PartialEq, Clone)] pub struct TokenWithLocation { @@ -445,7 +450,7 @@ impl<'a> Tokenizer<'a> { /// Tokenize the statement and produce a vector of tokens pub fn tokenize(&mut self) -> Result, TokenizerError> { - let twl = self.tokenize_with_location()?; + let (twl, _) = self.tokenize_with_location()?; let mut tokens: Vec = vec![]; tokens.reserve(twl.len()); @@ -456,7 +461,9 @@ impl<'a> Tokenizer<'a> { } /// Tokenize the statement and produce a vector of tokens with location information - pub fn tokenize_with_location(&mut self) -> Result, TokenizerError> { + pub fn tokenize_with_location( + &mut self, + ) -> Result<(Vec, Location), TokenizerError> { let mut state = State { peekable: self.query.chars().peekable(), line: 1, @@ -474,7 +481,7 @@ impl<'a> Tokenizer<'a> { location = state.location(); } - Ok(tokens) + Ok((tokens, state.location())) } /// Get the next token or return None @@ -1813,7 +1820,7 @@ mod tests { let sql = "SELECT a,\n b"; let dialect = GenericDialect {}; let mut tokenizer = Tokenizer::new(&dialect, sql); - let tokens = tokenizer.tokenize_with_location().unwrap(); + let (tokens, _) = tokenizer.tokenize_with_location().unwrap(); let expected = vec![ TokenWithLocation::new(Token::make_keyword("SELECT"), 1, 1), TokenWithLocation::new(Token::Whitespace(Whitespace::Space), 1, 7), diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index ada5fec5e..18547a3bd 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -17,6 +17,7 @@ use test_utils::*; use sqlparser::ast::*; use sqlparser::dialect::{BigQueryDialect, GenericDialect}; +use sqlparser::location::Located; #[test] fn parse_literal_string() { @@ -35,7 +36,7 @@ fn parse_literal_string() { #[test] fn parse_table_identifiers() { - fn test_table_ident(ident: &str, expected: Vec) { + fn test_table_ident(ident: &str, expected: Vec>) { let sql = format!("SELECT 1 FROM {}", ident); let select = bigquery().verified_only_select(&sql); assert_eq!( diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 3e974d56d..e11ce6809 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -35,10 +35,7 @@ fn parse_map_access_expr() { distinct: false, top: None, projection: vec![UnnamedExpr(MapAccess { - column: Box::new(Identifier(Ident { - value: "string_values".to_string(), - quote_style: None, - })), + column: Box::new(Identifier(Ident::new("string_values".to_string()))), keys: vec![Expr::Function(Function { name: ObjectName(vec!["indexOf".into()]), args: vec![ diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index c880eee7c..c4b4193ae 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -27,6 +27,7 @@ use sqlparser::dialect::{ MySqlDialect, PostgreSqlDialect, RedshiftSqlDialect, SQLiteDialect, SnowflakeDialect, }; use sqlparser::keywords::ALL_KEYWORDS; +use sqlparser::location::Located; use sqlparser::parser::{Parser, ParserError}; use test_utils::{ all_dialects, assert_eq_vec, expr_from_projection, join, number, only, table, table_alias, @@ -831,10 +832,7 @@ fn parse_select_with_date_column_name() { let sql = "SELECT date"; let select = verified_only_select(sql); assert_eq!( - &Expr::Identifier(Ident { - value: "date".into(), - quote_style: None, - }), + &Expr::Identifier(Ident::new("date")), expr_from_projection(only(&select.projection)), ); } @@ -1057,10 +1055,7 @@ fn parse_null_like() { pattern: Box::new(Expr::Value(Value::Null)), escape_char: None, }, - alias: Ident { - value: "col_null".to_owned(), - quote_style: None, - }, + alias: Ident::new("col_null"), }, select.projection[0] ); @@ -1072,10 +1067,7 @@ fn parse_null_like() { pattern: Box::new(Expr::Identifier(Ident::new("column1"))), escape_char: None, }, - alias: Ident { - value: "null_col".to_owned(), - quote_style: None, - }, + alias: Ident::new("null_col"), }, select.projection[1] ); @@ -1906,18 +1898,12 @@ fn parse_listagg() { }); let within_group = vec![ OrderByExpr { - expr: Expr::Identifier(Ident { - value: "id".to_string(), - quote_style: None, - }), + expr: Expr::Identifier(Ident::new("id")), asc: None, nulls_first: None, }, OrderByExpr { - expr: Expr::Identifier(Ident { - value: "username".to_string(), - quote_style: None, - }), + expr: Expr::Identifier(Ident::new("username")), asc: None, nulls_first: None, }, @@ -3444,17 +3430,11 @@ fn parse_interval_and_or_xor() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: false, top: None, - projection: vec![UnnamedExpr(Expr::Identifier(Ident { - value: "col".to_string(), - quote_style: None, - }))], + projection: vec![UnnamedExpr(Expr::Identifier(Ident::new("col")))], into: None, from: vec![TableWithJoins { relation: TableFactor::Table { - name: ObjectName(vec![Ident { - value: "test".to_string(), - quote_style: None, - }]), + name: ObjectName(vec![Ident::new("test")]), alias: None, args: None, with_hints: vec![], @@ -3464,16 +3444,10 @@ fn parse_interval_and_or_xor() { lateral_views: vec![], selection: Some(Expr::BinaryOp { left: Box::new(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "d3_date".to_string(), - quote_style: None, - })), + left: Box::new(Expr::Identifier(Ident::new("d3_date"))), op: BinaryOperator::Gt, right: Box::new(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "d1_date".to_string(), - quote_style: None, - })), + left: Box::new(Expr::Identifier(Ident::new("d1_date"))), op: BinaryOperator::Plus, right: Box::new(Expr::Interval { value: Box::new(Expr::Value(Value::SingleQuotedString( @@ -3488,16 +3462,10 @@ fn parse_interval_and_or_xor() { }), op: BinaryOperator::And, right: Box::new(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "d2_date".to_string(), - quote_style: None, - })), + left: Box::new(Expr::Identifier(Ident::new("d2_date"))), op: BinaryOperator::Gt, right: Box::new(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "d1_date".to_string(), - quote_style: None, - })), + left: Box::new(Expr::Identifier(Ident::new("d1_date"))), op: BinaryOperator::Plus, right: Box::new(Expr::Interval { value: Box::new(Expr::Value(Value::SingleQuotedString( @@ -3554,10 +3522,7 @@ fn parse_at_timezone() { assert_eq!( &Expr::AtTimeZone { timestamp: Box::new(Expr::Function(Function { - name: ObjectName(vec![Ident { - value: "FROM_UNIXTIME".to_string(), - quote_style: None, - }]), + name: ObjectName(vec![Ident::new("FROM_UNIXTIME")]), args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(zero.clone()))], over: None, distinct: false, @@ -3573,17 +3538,11 @@ fn parse_at_timezone() { assert_eq!( &SelectItem::ExprWithAlias { expr: Expr::Function(Function { - name: ObjectName(vec![Ident { - value: "DATE_FORMAT".to_string(), - quote_style: None, - },],), + name: ObjectName(vec![Ident::new("DATE_FORMAT")]), args: vec![ FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::AtTimeZone { timestamp: Box::new(Expr::Function(Function { - name: ObjectName(vec![Ident { - value: "FROM_UNIXTIME".to_string(), - quote_style: None, - },],), + name: ObjectName(vec![Ident::new("FROM_UNIXTIME")]), args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(zero))], over: None, distinct: false, @@ -3599,10 +3558,7 @@ fn parse_at_timezone() { distinct: false, special: false, },), - alias: Ident { - value: "hour".to_string(), - quote_style: Some('"'), - }, + alias: Ident::with_quote('"', "hour"), }, only(&select.projection), ); @@ -4283,14 +4239,8 @@ fn parse_recursive_cte() { assert_eq!(with.cte_tables.len(), 1); let expected = Cte { alias: TableAlias { - name: Ident { - value: "nums".to_string(), - quote_style: None, - }, - columns: vec![Ident { - value: "val".to_string(), - quote_style: None, - }], + name: Ident::new("nums"), + columns: vec![Ident::new("val")], }, query: Box::new(cte_query), from: None, @@ -4608,7 +4558,7 @@ fn parse_create_view() { cluster_by, } => { assert_eq!("myschema.myview", name.to_string()); - assert_eq!(Vec::::new(), columns); + assert_eq!(Vec::>::new(), columns); assert_eq!("SELECT foo FROM bar", query.to_string()); assert!(!materialized); assert!(!or_replace); @@ -4735,7 +4685,7 @@ fn parse_create_materialized_view() { cluster_by, } => { assert_eq!("myschema.myview", name.to_string()); - assert_eq!(Vec::::new(), columns); + assert_eq!(Vec::>::new(), columns); assert_eq!("SELECT foo FROM bar", query.to_string()); assert!(materialized); assert_eq!(with_options, vec![]); @@ -4760,7 +4710,7 @@ fn parse_create_materialized_view_with_cluster_by() { cluster_by, } => { assert_eq!("myschema.myview", name.to_string()); - assert_eq!(Vec::::new(), columns); + assert_eq!(Vec::>::new(), columns); assert_eq!("SELECT foo FROM bar", query.to_string()); assert!(materialized); assert_eq!(with_options, vec![]); @@ -5445,16 +5395,7 @@ fn parse_grant() { Action::Select { columns: None }, Action::Insert { columns: None }, Action::Update { - columns: Some(vec![ - Ident { - value: "shape".into(), - quote_style: None, - }, - Ident { - value: "size".into(), - quote_style: None, - }, - ]) + columns: Some(vec![Ident::new("shape"), Ident::new("size"),]) }, Action::Usage, Action::Delete, @@ -5673,10 +5614,7 @@ fn parse_merge() { locks: vec![], }), alias: Some(TableAlias { - name: Ident { - value: "stg".to_string(), - quote_style: None, - }, + name: Ident::new("stg"), columns: vec![], }), } @@ -5818,13 +5756,7 @@ fn test_lock_table() { assert_eq!(ast.locks.len(), 1); let lock = ast.locks.pop().unwrap(); assert_eq!(lock.lock_type, LockType::Update); - assert_eq!( - lock.of.unwrap().0, - vec![Ident { - value: "school".to_string(), - quote_style: None - }] - ); + assert_eq!(lock.of.unwrap().0, vec![Ident::new("school")]); assert!(lock.nonblock.is_none()); let sql = "SELECT * FROM student WHERE id = '1' FOR SHARE OF school"; @@ -5832,13 +5764,7 @@ fn test_lock_table() { assert_eq!(ast.locks.len(), 1); let lock = ast.locks.pop().unwrap(); assert_eq!(lock.lock_type, LockType::Share); - assert_eq!( - lock.of.unwrap().0, - vec![Ident { - value: "school".to_string(), - quote_style: None - }] - ); + assert_eq!(lock.of.unwrap().0, vec![Ident::new("school")]); assert!(lock.nonblock.is_none()); let sql = "SELECT * FROM student WHERE id = '1' FOR SHARE OF school FOR UPDATE OF student"; @@ -5846,23 +5772,11 @@ fn test_lock_table() { assert_eq!(ast.locks.len(), 2); let lock = ast.locks.remove(0); assert_eq!(lock.lock_type, LockType::Share); - assert_eq!( - lock.of.unwrap().0, - vec![Ident { - value: "school".to_string(), - quote_style: None - }] - ); + assert_eq!(lock.of.unwrap().0, vec![Ident::new("school")]); assert!(lock.nonblock.is_none()); let lock = ast.locks.remove(0); assert_eq!(lock.lock_type, LockType::Update); - assert_eq!( - lock.of.unwrap().0, - vec![Ident { - value: "student".to_string(), - quote_style: None - }] - ); + assert_eq!(lock.of.unwrap().0, vec![Ident::new("student")]); assert!(lock.nonblock.is_none()); } @@ -5873,13 +5787,7 @@ fn test_lock_nonblock() { assert_eq!(ast.locks.len(), 1); let lock = ast.locks.pop().unwrap(); assert_eq!(lock.lock_type, LockType::Update); - assert_eq!( - lock.of.unwrap().0, - vec![Ident { - value: "school".to_string(), - quote_style: None - }] - ); + assert_eq!(lock.of.unwrap().0, vec![Ident::new("school")]); assert_eq!(lock.nonblock.unwrap(), NonBlock::SkipLocked); let sql = "SELECT * FROM student WHERE id = '1' FOR SHARE OF school NOWAIT"; @@ -5887,13 +5795,7 @@ fn test_lock_nonblock() { assert_eq!(ast.locks.len(), 1); let lock = ast.locks.pop().unwrap(); assert_eq!(lock.lock_type, LockType::Share); - assert_eq!( - lock.of.unwrap().0, - vec![Ident { - value: "school".to_string(), - quote_style: None - }] - ); + assert_eq!(lock.of.unwrap().0, vec![Ident::new("school")]); assert_eq!(lock.nonblock.unwrap(), NonBlock::Nowait); } diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 41b0803e4..8dba7ee09 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -130,10 +130,7 @@ fn parse_mssql_create_role() { assert_eq_vec(&["mssql"], &names); assert_eq!( authorization_owner, - Some(ObjectName(vec![Ident { - value: "helena".into(), - quote_style: None - }])) + Some(ObjectName(vec![Ident::new("helena")])), ); } _ => unreachable!(), diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 508ae8461..50f1120cf 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -447,10 +447,9 @@ fn parse_quote_identifiers_2() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: false, top: None, - projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { - value: "quoted ` identifier".into(), - quote_style: Some('`'), - }))], + projection: vec![SelectItem::UnnamedExpr(Expr::Identifier( + Ident::with_quote('`', "quoted ` identifier") + ))], into: None, from: vec![], lateral_views: vec![], @@ -481,10 +480,9 @@ fn parse_quote_identifiers_3() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: false, top: None, - projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { - value: "`quoted identifier`".into(), - quote_style: Some('`'), - }))], + projection: vec![SelectItem::UnnamedExpr(Expr::Identifier( + Ident::with_quote('`', "`quoted identifier`") + ))], into: None, from: vec![], lateral_views: vec![], @@ -979,10 +977,7 @@ fn parse_substring_in_select() { distinct: true, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Substring { - expr: Box::new(Expr::Identifier(Ident { - value: "description".to_string(), - quote_style: None - })), + expr: Box::new(Expr::Identifier(Ident::new("description"))), substring_from: Some(Box::new(Expr::Value(Value::Number( "0".to_string(), false @@ -995,10 +990,7 @@ fn parse_substring_in_select() { into: None, from: vec![TableWithJoins { relation: TableFactor::Table { - name: ObjectName(vec![Ident { - value: "test".to_string(), - quote_style: None - }]), + name: ObjectName(vec![Ident::new("test")]), alias: None, args: None, with_hints: vec![] diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 496a61843..64c1a3fb3 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -790,10 +790,7 @@ fn parse_set() { local: false, hivevar: false, variable: ObjectName(vec![Ident::new("a")]), - value: vec![Expr::Identifier(Ident { - value: "b".into(), - quote_style: None - })], + value: vec![Expr::Identifier(Ident::new("b"))], } ); @@ -832,10 +829,7 @@ fn parse_set() { local: false, hivevar: false, variable: ObjectName(vec![Ident::new("a")]), - value: vec![Expr::Identifier(Ident { - value: "DEFAULT".into(), - quote_style: None - })], + value: vec![Expr::Identifier(Ident::new("DEFAULT"))], } ); @@ -857,10 +851,7 @@ fn parse_set() { local: false, hivevar: false, variable: ObjectName(vec![Ident::new("a"), Ident::new("b"), Ident::new("c")]), - value: vec![Expr::Identifier(Ident { - value: "b".into(), - quote_style: None - })], + value: vec![Expr::Identifier(Ident::new("b"))], } ); @@ -928,10 +919,7 @@ fn parse_set_role() { stmt, Statement::SetRole { context_modifier: ContextModifier::Local, - role_name: Some(Ident { - value: "rolename".to_string(), - quote_style: Some('\"'), - }), + role_name: Some(Ident::with_quote('\"', "rolename".to_string())), } ); assert_eq!(query, stmt.to_string()); @@ -942,10 +930,7 @@ fn parse_set_role() { stmt, Statement::SetRole { context_modifier: ContextModifier::None, - role_name: Some(Ident { - value: "rolename".to_string(), - quote_style: Some('\''), - }), + role_name: Some(Ident::with_quote('\'', "rolename".to_string())), } ); assert_eq!(query, stmt.to_string()); @@ -1117,7 +1102,7 @@ fn parse_pg_on_conflict() { })), .. } => { - assert_eq!(vec![Ident::from("did")], cols); + assert_eq!(vec![Located::::from("did")], cols); assert_eq!( OnConflictAction::DoUpdate(DoUpdate { assignments: vec![Assignment { @@ -1147,7 +1132,13 @@ fn parse_pg_on_conflict() { })), .. } => { - assert_eq!(vec![Ident::from("did"), Ident::from("area"),], cols); + assert_eq!( + vec![ + Located::::from("did"), + Located::::from("area"), + ], + cols + ); assert_eq!( OnConflictAction::DoUpdate(DoUpdate { assignments: vec![ @@ -1205,7 +1196,7 @@ fn parse_pg_on_conflict() { })), .. } => { - assert_eq!(vec![Ident::from("did")], cols); + assert_eq!(vec![Located::::from("did")], cols); assert_eq!( OnConflictAction::DoUpdate(DoUpdate { assignments: vec![Assignment { @@ -1213,10 +1204,7 @@ fn parse_pg_on_conflict() { value: Expr::Value(Value::Placeholder("$1".to_string())) },], selection: Some(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "dsize".to_string(), - quote_style: None - })), + left: Box::new(Expr::Identifier(Ident::new("dsize".to_string()))), op: BinaryOperator::Gt, right: Box::new(Expr::Value(Value::Placeholder("$2".to_string()))) }) @@ -1242,7 +1230,10 @@ fn parse_pg_on_conflict() { })), .. } => { - assert_eq!(vec![Ident::from("distributors_did_pkey")], cname.0); + assert_eq!( + vec![Located::::from("distributors_did_pkey")], + cname.0 + ); assert_eq!( OnConflictAction::DoUpdate(DoUpdate { assignments: vec![Assignment { @@ -1250,10 +1241,7 @@ fn parse_pg_on_conflict() { value: Expr::Value(Value::Placeholder("$1".to_string())) },], selection: Some(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "dsize".to_string(), - quote_style: None - })), + left: Box::new(Expr::Identifier(Ident::new("dsize".to_string()))), op: BinaryOperator::Gt, right: Box::new(Expr::Value(Value::Placeholder("$2".to_string()))) }) @@ -1443,14 +1431,8 @@ fn parse_array_index_expr() { obj: Box::new(Expr::Identifier(Ident::new("bar"))), indexes: vec![ num[0].clone(), - Expr::Identifier(Ident { - value: "baz".to_string(), - quote_style: Some('"') - }), - Expr::Identifier(Ident { - value: "fooz".to_string(), - quote_style: Some('"') - }) + Expr::Identifier(Ident::with_quote('"', "baz".to_string())), + Expr::Identifier(Ident::with_quote('"', "fooz".to_string())), ], }, expr_from_projection(only(&select.projection)), @@ -1684,7 +1666,7 @@ fn test_json() { let select = pg().verified_only_select(sql); assert_eq!( SelectItem::UnnamedExpr(Expr::JsonAccess { - left: Box::new(Expr::Identifier(Ident::from("info"))), + left: Box::new(Expr::Identifier(Located::::from("info"))), operator: JsonOperator::HashMinus, right: Box::new(Expr::Array(Array { elem: vec![ @@ -1701,7 +1683,7 @@ fn test_json() { let select = pg().verified_only_select(sql); assert_eq!( Expr::JsonAccess { - left: Box::new(Expr::Identifier(Ident::from("info"))), + left: Box::new(Expr::Identifier(Located::::from("info"))), operator: JsonOperator::AtQuestion, right: Box::new(Expr::Value(Value::SingleQuotedString("$.a".to_string())),), }, @@ -1712,7 +1694,7 @@ fn test_json() { let select = pg().verified_only_select(sql); assert_eq!( Expr::JsonAccess { - left: Box::new(Expr::Identifier(Ident::from("info"))), + left: Box::new(Expr::Identifier(Located::::from("info"))), operator: JsonOperator::AtAt, right: Box::new(Expr::Value(Value::SingleQuotedString("$.a".to_string())),), }, @@ -1996,10 +1978,7 @@ fn parse_custom_operator() { assert_eq!( select.selection, Some(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "relname".into(), - quote_style: None, - })), + left: Box::new(Expr::Identifier(Ident::new("relname"))), op: BinaryOperator::PGCustomBinaryOperator(vec![ "database".into(), "pg_catalog".into(), @@ -2015,10 +1994,7 @@ fn parse_custom_operator() { assert_eq!( select.selection, Some(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "relname".into(), - quote_style: None, - })), + left: Box::new(Expr::Identifier(Ident::new("relname"))), op: BinaryOperator::PGCustomBinaryOperator(vec!["pg_catalog".into(), "~".into()]), right: Box::new(Expr::Value(Value::SingleQuotedString("^(table)$".into()))) }) @@ -2030,10 +2006,7 @@ fn parse_custom_operator() { assert_eq!( select.selection, Some(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident { - value: "relname".into(), - quote_style: None, - })), + left: Box::new(Expr::Identifier(Ident::new("relname"))), op: BinaryOperator::PGCustomBinaryOperator(vec!["~".into()]), right: Box::new(Expr::Value(Value::SingleQuotedString("^(table)$".into()))) }) @@ -2426,10 +2399,7 @@ fn parse_drop_function() { Statement::DropFunction { if_exists: true, func_desc: vec![DropFunctionDesc { - name: ObjectName(vec![Ident { - value: "test_func".to_string(), - quote_style: None - }]), + name: ObjectName(vec![Ident::new("test_func".to_string())]), args: None }], option: None @@ -2442,10 +2412,7 @@ fn parse_drop_function() { Statement::DropFunction { if_exists: true, func_desc: vec![DropFunctionDesc { - name: ObjectName(vec![Ident { - value: "test_func".to_string(), - quote_style: None - }]), + name: ObjectName(vec![Ident::new("test_func".to_string())]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), OperateFunctionArg { @@ -2467,10 +2434,7 @@ fn parse_drop_function() { if_exists: true, func_desc: vec![ DropFunctionDesc { - name: ObjectName(vec![Ident { - value: "test_func1".to_string(), - quote_style: None - }]), + name: ObjectName(vec![Ident::new("test_func1".to_string())]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), OperateFunctionArg { @@ -2485,10 +2449,7 @@ fn parse_drop_function() { ]), }, DropFunctionDesc { - name: ObjectName(vec![Ident { - value: "test_func2".to_string(), - quote_style: None - }]), + name: ObjectName(vec![Ident::new("test_func2")]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Varchar(None)), OperateFunctionArg { @@ -2553,10 +2514,7 @@ fn parse_dollar_quoted_string() { tag: None, value: "Foo$Bar".into(), })), - alias: Ident { - value: "col_name".into(), - quote_style: None, - }, + alias: Ident::new("col_name".to_string()), } ); diff --git a/tests/sqlparser_redshift.rs b/tests/sqlparser_redshift.rs index 7597ee981..9f69dc59c 100644 --- a/tests/sqlparser_redshift.rs +++ b/tests/sqlparser_redshift.rs @@ -23,24 +23,15 @@ fn test_square_brackets_over_db_schema_table_name() { let select = redshift().verified_only_select("SELECT [col1] FROM [test_schema].[test_table]"); assert_eq!( select.projection[0], - SelectItem::UnnamedExpr(Expr::Identifier(Ident { - value: "col1".to_string(), - quote_style: Some('[') - })), + SelectItem::UnnamedExpr(Expr::Identifier(Ident::with_quote('[', "col1"))), ); assert_eq!( select.from[0], TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![ - Ident { - value: "test_schema".to_string(), - quote_style: Some('[') - }, - Ident { - value: "test_table".to_string(), - quote_style: Some('[') - } + Ident::with_quote('[', "test_schema"), + Ident::with_quote('[', "test_table"), ]), alias: None, args: None, @@ -67,24 +58,15 @@ fn test_double_quotes_over_db_schema_table_name() { redshift().verified_only_select("SELECT \"col1\" FROM \"test_schema\".\"test_table\""); assert_eq!( select.projection[0], - SelectItem::UnnamedExpr(Expr::Identifier(Ident { - value: "col1".to_string(), - quote_style: Some('"') - })), + SelectItem::UnnamedExpr(Expr::Identifier(Ident::with_quote('"', "col1"))), ); assert_eq!( select.from[0], TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![ - Ident { - value: "test_schema".to_string(), - quote_style: Some('"') - }, - Ident { - value: "test_table".to_string(), - quote_style: Some('"') - } + Ident::with_quote('"', "test_schema"), + Ident::with_quote('"', "test_table"), ]), alias: None, args: None,