chore: replace Tokenizer with databend-common-ast (#450)

cli/Cargo.toml

@@ -21,16 +21,13 @@ chrono = { version = "0.4.35", default-features = false, features = ["clock"] }
 clap = { version = "4.4", features = ["derive", "env"] }
 comfy-table = "7.1"
 csv = "1.3"
-databend-common-ast = "0.0.1"
+databend-common-ast = "0.0.2"
 fern = { version = "0.6", features = ["colored"] }
 indicatif = "0.17"
 log = "0.4"
-logos = "0.13"
 once_cell = "1.18"
 rustyline = "12.0"
 serde = { version = "1.0", features = ["derive"] }
-strum = "0.25"
-strum_macros = "0.25"
 terminal_size = "0.3"
 tokio = { version = "1.34", features = [
     "macros",

cli/src/ast/mod.rs

@@ -12,13 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod tokenizer;
-
 use databend_common_ast::{
     ast::pretty_statement,
     parser::{parse_sql, Dialect},
 };
-pub use tokenizer::*;
 
 use crate::session::QueryKind;