From e22a4303829ee8fa50db804624d285b6b2896c30 Mon Sep 17 00:00:00 2001 From: Sam Cao Date: Mon, 18 Mar 2024 11:53:41 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20tokens=20to=20parse?= =?UTF-8?q?=20output=20for=20JNI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rust/src/core.rs | 5 +-- rust/src/outputs.rs | 19 +++++++++++ rust/tests/test_core.rs | 73 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/rust/src/core.rs b/rust/src/core.rs index 1a61a631..392bccbc 100644 --- a/rust/src/core.rs +++ b/rust/src/core.rs @@ -35,10 +35,7 @@ pub fn parse<'local>(code: String, options: options::ParseOptions) -> Result parse_module(parse_params), }; match result { - Ok(parsed_source) => Ok(outputs::ParseOutput { - module: parsed_source.is_module(), - script: parsed_source.is_script(), - }), + Ok(parsed_source) => Ok(outputs::ParseOutput::new(parsed_source, options.capture_tokens)), Err(e) => Err(e.to_string()), } } diff --git a/rust/src/outputs.rs b/rust/src/outputs.rs index 727a547c..4158a5d8 100644 --- a/rust/src/outputs.rs +++ b/rust/src/outputs.rs @@ -19,6 +19,9 @@ use jni::objects::{GlobalRef, JMethodID, JObject}; use jni::sys::jvalue; use jni::JNIEnv; +use deno_ast::swc::parser::token::TokenAndSpan; +use deno_ast::ParsedSource; + use std::ptr::null_mut; use crate::converter; @@ -126,6 +129,22 @@ pub trait ToJniType { pub struct ParseOutput { pub module: bool, pub script: bool, + pub tokens: Option>, +} + +impl ParseOutput { + pub fn new(parsed_source: ParsedSource, capture_tokens: bool) -> ParseOutput { + let tokens = if capture_tokens { + Some(parsed_source.tokens().to_vec()) + } else { + None + }; + ParseOutput { + module: parsed_source.is_module(), + script: parsed_source.is_script(), + tokens, + } + } } impl ToJniType for ParseOutput { diff --git a/rust/tests/test_core.rs b/rust/tests/test_core.rs index 3b18f482..fec9a961 100644 --- a/rust/tests/test_core.rs +++ b/rust/tests/test_core.rs @@ -15,7 +15,9 @@ * limitations under the License. */ -use deno_ast::MediaType; +use deno_ast::{ + swc::{atoms::JsWord, common::Spanned, common::BytePos, parser::token::{IdentLike, Keyword, Token, Word}}, MediaType, +}; use swc4j::*; @@ -45,6 +47,75 @@ fn test_parse_jsx_with_default_options() { assert!(!output.script); } +#[test] +fn test_parse_typescript_with_default_options() { + let code = "function add(a:number, b:number) { return a+b; }"; + let options = options::ParseOptions { + media_type: MediaType::TypeScript, + ..Default::default() + }; + let output = core::parse(code.to_owned(), options); + assert!(output.is_ok()); + let output = output.unwrap(); + assert!(output.module); + assert!(!output.script); + assert!(output.tokens.is_none()); +} + +#[test] +fn test_parse_typescript_with_capture_tokens() { + let code = "function add(a:number, b:number) { return a+b; }"; + let options = options::ParseOptions { + capture_tokens: true, + media_type: MediaType::TypeScript, + ..Default::default() + }; + let output = core::parse(code.to_owned(), options); + assert!(output.is_ok()); + let output = output.unwrap(); + assert!(output.module); + assert!(!output.script); + assert!(output.tokens.is_some()); + let tokens = output.tokens.unwrap(); + /* + * TokenAndSpan { token: function, had_line_break: true, span: Span { lo: BytePos(1), hi: BytePos(9), ctxt: #0 } } + * TokenAndSpan { token: add, had_line_break: false, span: Span { lo: BytePos(10), hi: BytePos(13), ctxt: #0 } } + * TokenAndSpan { token: (, had_line_break: false, span: Span { lo: BytePos(13), hi: BytePos(14), ctxt: #0 } } + * TokenAndSpan { token: a, had_line_break: false, span: Span { lo: BytePos(14), hi: BytePos(15), ctxt: #0 } } + * TokenAndSpan { token: :, had_line_break: false, span: Span { lo: BytePos(15), hi: BytePos(16), ctxt: #0 } } + * TokenAndSpan { token: number, had_line_break: false, span: Span { lo: BytePos(16), hi: BytePos(22), ctxt: #0 } } + * TokenAndSpan { token: ,, had_line_break: false, span: Span { lo: BytePos(22), hi: BytePos(23), ctxt: #0 } } + * TokenAndSpan { token: b, had_line_break: false, span: Span { lo: BytePos(24), hi: BytePos(25), ctxt: #0 } } + * TokenAndSpan { token: :, had_line_break: false, span: Span { lo: BytePos(25), hi: BytePos(26), ctxt: #0 } } + * TokenAndSpan { token: number, had_line_break: false, span: Span { lo: BytePos(26), hi: BytePos(32), ctxt: #0 } } + * TokenAndSpan { token: ), had_line_break: false, span: Span { lo: BytePos(32), hi: BytePos(33), ctxt: #0 } } + * TokenAndSpan { token: {, had_line_break: false, span: Span { lo: BytePos(34), hi: BytePos(35), ctxt: #0 } } + * TokenAndSpan { token: return, had_line_break: false, span: Span { lo: BytePos(36), hi: BytePos(42), ctxt: #0 } } + * TokenAndSpan { token: a, had_line_break: false, span: Span { lo: BytePos(43), hi: BytePos(44), ctxt: #0 } } + * TokenAndSpan { token: +, had_line_break: false, span: Span { lo: BytePos(44), hi: BytePos(45), ctxt: #0 } } + * TokenAndSpan { token: b, had_line_break: false, span: Span { lo: BytePos(45), hi: BytePos(46), ctxt: #0 } } + * TokenAndSpan { token: ;, had_line_break: false, span: Span { lo: BytePos(46), hi: BytePos(47), ctxt: #0 } } + * TokenAndSpan { token: }, had_line_break: false, span: Span { lo: BytePos(48), hi: BytePos(49), ctxt: #0 } } + */ + let t0 = &tokens[0]; + let t1 = &tokens[1]; + let t2 = &tokens[2]; + let t3 = &tokens[3]; + let t12 = &tokens[12]; + assert_eq!(Token::Word(Word::Keyword(Keyword::Function)), t0.token); + assert_eq!(Token::Word(Word::Ident(IdentLike::Other(JsWord::new("add")))), t1.token); + assert_eq!(Token::LParen, t2.token); + assert_eq!(Token::Word(Word::Ident(IdentLike::Other(JsWord::new("a")))), t3.token); + assert_eq!(Token::Word(Word::Keyword(Keyword::Return)), t12.token); + assert!(t0.had_line_break); + assert!(!t1.had_line_break); + assert!(!t2.had_line_break); + assert!(!t3.had_line_break); + assert!(!t12.had_line_break); + assert_eq!(BytePos(1), t0.span_lo()); + assert_eq!(BytePos(9), t0.span_hi()); +} + #[test] fn test_parse_wrong_media_type() { let code = "function add(a:number, b:number) { return a+b; }";