feat: implement create external table parser (GreptimeTeam#1252)

* refactor: move parse_option_string to util

* feat: implement create external table parser

src/datanode/src/instance/sql.rs

@@ -90,6 +90,9 @@ impl Instance {
                     .execute(SqlRequest::CreateTable(request), query_ctx)
                     .await
             }
+            QueryStatement::Sql(Statement::CreateExternalTable(_create_external_table)) => {
+                unimplemented!()
+            }
             QueryStatement::Sql(Statement::Alter(alter_table)) => {
                 let name = alter_table.table_name().clone();
                 let (catalog, schema, table) = table_idents_to_full_name(&name, query_ctx.clone())?;

src/frontend/src/instance.rs

@@ -480,6 +480,7 @@ impl Instance {
 
             Statement::Tql(tql) => self.execute_tql(tql, query_ctx).await,
             Statement::CreateDatabase(_)
+            | Statement::CreateExternalTable(_)
             | Statement::ShowDatabases(_)
             | Statement::CreateTable(_)
             | Statement::ShowTables(_)
@@ -663,7 +664,8 @@ pub fn check_permission(
         // database ops won't be checked
         Statement::CreateDatabase(_) | Statement::ShowDatabases(_) | Statement::Use(_) => {}
         // show create table and alter are not supported yet
-        Statement::ShowCreateTable(_) | Statement::Alter(_) => {}
+        Statement::ShowCreateTable(_) | Statement::CreateExternalTable(_) | Statement::Alter(_) => {
+        }
 
         Statement::Insert(insert) => {
             validate_param(insert.table_name(), query_ctx)?;

src/sql/src/lib.rs

@@ -20,3 +20,4 @@ pub mod error;
 pub mod parser;
 pub mod parsers;
 pub mod statements;
+mod util;

src/sql/src/parsers/copy_parser.rs

@@ -13,13 +13,14 @@
 // limitations under the License.
 
 use snafu::ResultExt;
-use sqlparser::ast::{ObjectName, Value};
+use sqlparser::ast::ObjectName;
 use sqlparser::keywords::Keyword;
 
 use crate::error::{self, Result};
 use crate::parser::ParserContext;
 use crate::statements::copy::{CopyTable, CopyTableArgument, Format};
 use crate::statements::statement::Statement;
+use crate::util::parse_option_string;
 
 // COPY tbl TO 'output.parquet';
 impl<'a> ParserContext<'a> {
@@ -70,12 +71,12 @@ impl<'a> ParserContext<'a> {
         for option in options {
             match option.name.value.to_ascii_uppercase().as_str() {
                 "FORMAT" => {
-                    if let Some(fmt_str) = ParserContext::parse_option_string(option.value) {
+                    if let Some(fmt_str) = parse_option_string(option.value) {
                         format = Format::try_from(fmt_str)?;
                     }
                 }
                 "PATTERN" => {
-                    if let Some(v) = ParserContext::parse_option_string(option.value) {
+                    if let Some(v) = parse_option_string(option.value) {
                         pattern = Some(v);
                     }
                 }
@@ -92,7 +93,7 @@ impl<'a> ParserContext<'a> {
         let connection = connection_options
             .into_iter()
             .filter_map(|option| {
-                if let Some(v) = ParserContext::parse_option_string(option.value) {
+                if let Some(v) = parse_option_string(option.value) {
                     Some((option.name.value.to_uppercase(), v))
                 } else {
                     None
@@ -127,7 +128,7 @@ impl<'a> ParserContext<'a> {
         let mut format = Format::Parquet;
         for option in options {
             if option.name.value.eq_ignore_ascii_case("FORMAT") {
-                if let Some(fmt_str) = ParserContext::parse_option_string(option.value) {
+                if let Some(fmt_str) = parse_option_string(option.value) {
                     format = Format::try_from(fmt_str)?;
                 }
             }
@@ -141,7 +142,7 @@ impl<'a> ParserContext<'a> {
         let connection = connection_options
             .into_iter()
             .filter_map(|option| {
-                if let Some(v) = ParserContext::parse_option_string(option.value) {
+                if let Some(v) = parse_option_string(option.value) {
                     Some((option.name.value.to_uppercase(), v))
                 } else {
                     None
@@ -157,13 +158,6 @@ impl<'a> ParserContext<'a> {
             location,
         })
     }
-
-    fn parse_option_string(value: Value) -> Option<String> {
-        match value {
-            Value::SingleQuotedString(v) | Value::DoubleQuotedString(v) => Some(v),
-            _ => None,
-        }
-    }
 }
 
 #[cfg(test)]

src/sql/src/parsers/create_parser.rs

@@ -31,10 +31,11 @@ use crate::error::{
 };
 use crate::parser::ParserContext;
 use crate::statements::create::{
-    CreateDatabase, CreateTable, PartitionEntry, Partitions, TIME_INDEX,
+    CreateDatabase, CreateExternalTable, CreateTable, PartitionEntry, Partitions, TIME_INDEX,
 };
 use crate::statements::statement::Statement;
 use crate::statements::{sql_data_type_to_concrete_data_type, sql_value_to_value};
+use crate::util::parse_option_string;
 
 const ENGINE: &str = "ENGINE";
 const MAXVALUE: &str = "MAXVALUE";
@@ -51,12 +52,53 @@ impl<'a> ParserContext<'a> {
 
                 Keyword::SCHEMA | Keyword::DATABASE => self.parse_create_database(),
 
+                Keyword::EXTERNAL => self.parse_create_external_table(),
+
                 _ => self.unsupported(w.to_string()),
             },
             unexpected => self.unsupported(unexpected.to_string()),
         }
     }
 
+    fn parse_create_external_table(&mut self) -> Result<Statement> {
+        self.parser.next_token();
+        self.parser
+            .expect_keyword(Keyword::TABLE)
+            .context(error::SyntaxSnafu { sql: self.sql })?;
+
+        let table_name = self
+            .parser
+            .parse_object_name()
+            .context(error::UnexpectedSnafu {
+                sql: self.sql,
+                expected: "a table name",
+                actual: self.peek_token_as_string(),
+            })?;
+
+        let (columns, constraints) = self.parse_columns()?;
+
+        let options = self
+            .parser
+            .parse_options(Keyword::WITH)
+            .context(error::SyntaxSnafu { sql: self.sql })?
+            .into_iter()
+            .filter_map(|option| {
+                if let Some(v) = parse_option_string(option.value) {
+                    Some((option.name.value.to_uppercase(), v))
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        Ok(Statement::CreateExternalTable(CreateExternalTable {
+            name: table_name,
+            columns,
+            constraints,
+            options,
+        }))
+    }
+
     fn parse_create_database(&mut self) -> Result<Statement> {
         self.parser.next_token();
 
@@ -725,12 +767,92 @@ fn ensure_partition_names_no_duplicate(partitions: &Partitions) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use std::assert_matches::assert_matches;
+    use std::collections::HashMap;
 
     use sqlparser::ast::ColumnOption::NotNull;
     use sqlparser::dialect::GenericDialect;
 
     use super::*;
 
+    #[test]
+    fn test_parse_create_external_table() {
+        struct Test<'a> {
+            sql: &'a str,
+            expected_table_name: &'a str,
+            expected_options: HashMap<String, String>,
+        }
+
+        let tests = [Test {
+            sql: "CREATE EXTERNAL TABLE city with(location='/var/data/city.csv',format='csv');",
+            expected_table_name: "city",
+            expected_options: HashMap::from([
+                ("LOCATION".to_string(), "/var/data/city.csv".to_string()),
+                ("FORMAT".to_string(), "csv".to_string()),
+            ]),
+        }];
+
+        for test in tests {
+            let stmts = ParserContext::create_with_dialect(test.sql, &GenericDialect {}).unwrap();
+            assert_eq!(1, stmts.len());
+            match &stmts[0] {
+                Statement::CreateExternalTable(c) => {
+                    assert_eq!(c.name.to_string(), test.expected_table_name.to_string());
+                    assert_eq!(c.options, test.expected_options);
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_parse_create_external_table_with_schema() {
+        let sql = "CREATE EXTERNAL TABLE city (
+            host string,
+            ts int64,
+            cpu float64 default 0,
+            memory float64,
+            TIME INDEX (ts),
+            PRIMARY KEY(ts, host)
+        ) with(location='/var/data/city.csv',format='csv');";
+
+        let options = HashMap::from([
+            ("LOCATION".to_string(), "/var/data/city.csv".to_string()),
+            ("FORMAT".to_string(), "csv".to_string()),
+        ]);
+
+        let stmts = ParserContext::create_with_dialect(sql, &GenericDialect {}).unwrap();
+        assert_eq!(1, stmts.len());
+        match &stmts[0] {
+            Statement::CreateExternalTable(c) => {
+                assert_eq!(c.name.to_string(), "city");
+                assert_eq!(c.options, options);
+
+                let columns = &c.columns;
+                assert_column_def(&columns[0], "host", "STRING");
+                assert_column_def(&columns[1], "ts", "int64");
+                assert_column_def(&columns[2], "cpu", "float64");
+                assert_column_def(&columns[3], "memory", "float64");
+
+                let constraints = &c.constraints;
+                assert_matches!(
+                    &constraints[0],
+                    TableConstraint::Unique {
+                        is_primary: false,
+                        ..
+                    }
+                );
+                assert_matches!(
+                    &constraints[1],
+                    TableConstraint::Unique {
+                        is_primary: true,
+                        ..
+                    }
+                );
+            }
+            _ => unreachable!(),
+        }
+    }
+
     #[test]
     fn test_parse_create_database() {
         let sql = "create database";

src/sql/src/statements/create.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
+
 use crate::ast::{ColumnDef, Ident, ObjectName, SqlOption, TableConstraint, Value as SqlValue};
 
 /// Time index name, used in table constraints.
@@ -50,3 +52,13 @@ pub struct CreateDatabase {
     /// Create if not exists
     pub if_not_exists: bool,
 }
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct CreateExternalTable {
+    /// Table name
+    pub name: ObjectName,
+    pub columns: Vec<ColumnDef>,
+    pub constraints: Vec<TableConstraint>,
+    /// Table options in `WITH`.
+    pub options: HashMap<String, String>,
+}

src/sql/src/statements/statement.rs

@@ -18,7 +18,7 @@ use sqlparser::ast::Statement as SpStatement;
 use crate::error::{ConvertToDfStatementSnafu, Error};
 use crate::statements::alter::AlterTable;
 use crate::statements::copy::CopyTable;
-use crate::statements::create::{CreateDatabase, CreateTable};
+use crate::statements::create::{CreateDatabase, CreateExternalTable, CreateTable};
 use crate::statements::delete::Delete;
 use crate::statements::describe::DescribeTable;
 use crate::statements::drop::DropTable;
@@ -40,6 +40,8 @@ pub enum Statement {
     Delete(Box<Delete>),
     /// CREATE TABLE
     CreateTable(CreateTable),
+    // CREATE EXTERNAL TABLE
+    CreateExternalTable(CreateExternalTable),
     // DROP TABLE
     DropTable(DropTable),
     // CREATE DATABASE

src/sql/src/util.rs

@@ -0,0 +1,22 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sqlparser::ast::Value;
+
+pub fn parse_option_string(value: Value) -> Option<String> {
+    match value {
+        Value::SingleQuotedString(v) | Value::DoubleQuotedString(v) => Some(v),
+        _ => None,
+    }
+}