General small fixes (#70)

.github/workflows/github-release.yml

@@ -20,6 +20,7 @@ jobs:
       sponsors: ${{ steps.get-sponsors.outputs.sponsors }}
 
     steps:
+      - uses: actions/checkout@v3
       - name: Get version
         id: get-version
         run: |

parser/src/expressions/mod.rs

@@ -1,7 +1,7 @@
 use crate::{
 	declarations::ClassDeclaration,
 	errors::parse_lexing_error,
-	functions::GeneralFunctionBase,
+	functions,
 	operators::{
 		AssociativityDirection, BinaryAssignmentOperator, UnaryPostfixAssignmentOperator,
 		UnaryPrefixAssignmentOperator, ASSIGNMENT_PRECEDENCE, AS_PRECEDENCE,
@@ -46,7 +46,7 @@ pub use arrow_function::{ArrowFunction, ExpressionOrBlock};
 
 pub use template_literal::{TemplateLiteral, TemplateLiteralPart};
 
-pub type ExpressionFunctionBase = GeneralFunctionBase<ExpressionPosition>;
+pub type ExpressionFunctionBase = functions::GeneralFunctionBase<ExpressionPosition>;
 pub type ExpressionFunction = FunctionBase<ExpressionFunctionBase>;
 
 use std::convert::{TryFrom, TryInto};
@@ -515,11 +515,15 @@ impl Expression {
 			}
 			t @ Token(TSXToken::Keyword(TSXKeyword::Function), _) => {
 				let span = t.get_span();
+				let generator_star_token_position = reader
+					.conditional_next(|tok| matches!(tok, TSXToken::Multiply))
+					.map(|token| token.get_span());
 				let header = FunctionHeader::VirginFunctionHeader {
 					async_keyword: None,
 					function_keyword: Keyword::new(span.clone()),
-					// TODO
-					generator_star_token_position: None,
+					generator_star_token_position,
+					#[cfg(feature = "extras")]
+					location: None,
 					position: span,
 				};
 				let name = if let Some(Token(TSXToken::OpenParentheses, _)) = reader.peek() {
@@ -534,7 +538,6 @@ impl Expression {
 				)?;
 				Expression::ExpressionFunction(function)
 			}
-			// TODO this should be extracted to a function that allows it to also work for leading `generator`
 			t @ Token(TSXToken::Keyword(TSXKeyword::Async), _) => {
 				let async_keyword = Some(Keyword::new(t.get_span()));
 				let header = crate::functions::function_header_from_reader_with_async_keyword(
@@ -553,13 +556,34 @@ impl Expression {
 				)?)
 			}
 			#[cfg(feature = "extras")]
-			t @ Token(TSXToken::Keyword(TSXKeyword::Generator), _) => {
-				let get_span = t.get_span();
+			Token(
+				TSXToken::Keyword(
+					kw @ TSXKeyword::Generator | kw @ TSXKeyword::Module | kw @ TSXKeyword::Server,
+				),
+				start,
+			) => {
+				// TODO bad way of doing this
+				use enum_variants_strings::EnumVariantsStrings;
+				let pos = start.with_length(kw.to_str().len());
+
+				let (generator_keyword, location) = match kw {
+					TSXKeyword::Generator => {
+						(Some(Keyword::new(pos)), functions::parse_function_location(reader))
+					}
+					TSXKeyword::Module => {
+						(None, Some(functions::FunctionLocationModifier::Module(Keyword::new(pos))))
+					}
+					TSXKeyword::Server => {
+						(None, Some(functions::FunctionLocationModifier::Server(Keyword::new(pos))))
+					}
+					_ => unreachable!(),
+				};
 				let function_keyword = Keyword::from_reader(reader)?;
-				let position = get_span.union(function_keyword.get_position());
+				let position = start.union(function_keyword.get_position());
 				let header = FunctionHeader::ChadFunctionHeader {
 					async_keyword: None,
-					generator_keyword: Some(Keyword::new(get_span)),
+					location,
+					generator_keyword,
 					function_keyword,
 					position,
 				};
@@ -1760,7 +1784,8 @@ pub enum SuperReference {
 mod tests {
 	use super::{ASTNode, Expression, Expression::*, MultipleExpression};
 	use crate::{
-		assert_matches_ast, operators::BinaryOperator, span, NumberRepresentation, Quoted, SourceId,
+		assert_matches_ast, ast::SpreadExpression, operators::BinaryOperator, span,
+		NumberRepresentation, Quoted, SourceId,
 	};
 
 	#[test]
@@ -1825,6 +1850,14 @@ mod tests {
 		);
 	}
 
+	#[test]
+	fn spread_function_argument() {
+		assert_matches_ast!(
+			"console.table(...a)",
+			FunctionCall { arguments: Deref @ [SpreadExpression::Spread(VariableReference(..), span!(14, 18))], .. }
+		);
+	}
+
 	#[test]
 	fn binary_expressions() {
 		assert_matches_ast!("2 + 3", BinaryOperation {

parser/src/functions.rs

@@ -207,6 +207,7 @@ where
 	}
 }
 
+/// Base for all functions with the `function` keyword
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct GeneralFunctionBase<T: ExpressionOrStatementPosition>(PhantomData<T>);
 
@@ -245,12 +246,23 @@ impl<T: ExpressionOrStatementPosition> FunctionBased for GeneralFunctionBase<T>
 	}
 }
 
+#[cfg(feature = "extras")]
+#[derive(Debug, PartialEq, Eq, Clone)]
+#[cfg_attr(feature = "self-rust-tokenize", derive(self_rust_tokenize::SelfRustTokenize))]
+#[cfg_attr(feature = "serde-serialize", derive(serde::Serialize))]
+pub enum FunctionLocationModifier {
+	Server(Keyword<tsx_keywords::Server>),
+	Module(Keyword<tsx_keywords::Module>),
+}
+
 #[derive(Debug, PartialEq, Eq, Clone)]
 #[cfg_attr(feature = "self-rust-tokenize", derive(self_rust_tokenize::SelfRustTokenize))]
 #[cfg_attr(feature = "serde-serialize", derive(serde::Serialize))]
 pub enum FunctionHeader {
 	VirginFunctionHeader {
 		async_keyword: Option<Keyword<tsx_keywords::Async>>,
+		#[cfg(feature = "extras")]
+		location: Option<FunctionLocationModifier>,
 		function_keyword: Keyword<tsx_keywords::Function>,
 		generator_star_token_position: Option<Span>,
 		position: Span,
@@ -259,6 +271,7 @@ pub enum FunctionHeader {
 	ChadFunctionHeader {
 		async_keyword: Option<Keyword<tsx_keywords::Async>>,
 		generator_keyword: Option<Keyword<tsx_keywords::Generator>>,
+		location: Option<FunctionLocationModifier>,
 		function_keyword: Keyword<tsx_keywords::Function>,
 		position: Span,
 	},
@@ -313,6 +326,8 @@ pub(crate) fn function_header_from_reader_with_async_keyword(
 		if let Some(token) = next_generator {
 			let span = token.get_span();
 			let generator_keyword = Some(Keyword::new(span.clone()));
+			let location = parse_function_location(reader);
+
 			let function_keyword = Keyword::from_reader(reader)?;
 			let position = async_keyword
 				.as_ref()
@@ -321,6 +336,7 @@ pub(crate) fn function_header_from_reader_with_async_keyword(
 
 			Ok(FunctionHeader::ChadFunctionHeader {
 				async_keyword,
+				location,
 				generator_keyword,
 				function_keyword,
 				position,
@@ -329,14 +345,41 @@ pub(crate) fn function_header_from_reader_with_async_keyword(
 			parse_regular_header(reader, async_keyword)
 		}
 	}
+
 	#[cfg(not(feature = "extras"))]
 	parse_regular_header(reader, async_keyword)
 }
 
+#[cfg(feature = "extras")]
+pub(crate) fn parse_function_location(
+	reader: &mut impl TokenReader<TSXToken, source_map::Start>,
+) -> Option<FunctionLocationModifier> {
+	use crate::{TSXKeyword, Token};
+
+	if let Some(Token(TSXToken::Keyword(TSXKeyword::Server | TSXKeyword::Module), _)) =
+		reader.peek()
+	{
+		Some(match reader.next().unwrap() {
+			t @ Token(TSXToken::Keyword(TSXKeyword::Server), _) => {
+				FunctionLocationModifier::Server(Keyword::new(t.get_span()))
+			}
+			t @ Token(TSXToken::Keyword(TSXKeyword::Module), _) => {
+				FunctionLocationModifier::Module(Keyword::new(t.get_span()))
+			}
+			_ => unreachable!(),
+		})
+	} else {
+		None
+	}
+}
+
 fn parse_regular_header(
 	reader: &mut impl TokenReader<TSXToken, source_map::Start>,
 	async_keyword: Option<Keyword<tsx_keywords::Async>>,
 ) -> Result<FunctionHeader, crate::ParseError> {
+	#[cfg(feature = "extras")]
+	let location = parse_function_location(reader);
+
 	let function_keyword = Keyword::from_reader(reader)?;
 	let generator_star_token_position = reader
 		.conditional_next(|tok| matches!(tok, TSXToken::Multiply))
@@ -356,6 +399,8 @@ fn parse_regular_header(
 		function_keyword,
 		generator_star_token_position,
 		position,
+		#[cfg(feature = "extras")]
+		location,
 	})
 }
 

parser/src/lexer.rs

@@ -290,9 +290,28 @@ pub fn lex_script(
 							if script[..idx].ends_with('_') {
 								return_err!(LexingErrors::InvalidUnderscore)
 							} else if *fractional {
-								return_err!(LexingErrors::SecondDecimalPoint);
+								// Catch for spread token `...`
+								if start + 1 == idx {
+									let automaton = TSXToken::new_automaton();
+									let derive_finite_automaton::GetNextResult::NewState(
+										dot_state_one,
+									) = automaton.get_next('.')
+									else {
+										unreachable!()
+									};
+									let derive_finite_automaton::GetNextResult::NewState(
+										dot_state_two,
+									) = dot_state_one.get_next('.')
+									else {
+										unreachable!()
+									};
+									state = LexingState::Symbol(dot_state_two);
+								} else {
+									return_err!(LexingErrors::SecondDecimalPoint);
+								}
+							} else {
+								*fractional = true;
 							}
-							*fractional = true;
 						} else {
 							return_err!(LexingErrors::NumberLiteralCannotHaveDecimalPoint);
 						}

parser/src/lib.rs

@@ -82,17 +82,23 @@ impl Quoted {
 pub struct ParseOptions {
 	/// Parsing of [JSX](https://facebook.github.io/jsx/) (includes some additions)
 	pub jsx: bool,
+	/// Allow custom characters in JSX attributes
 	pub special_jsx_attributes: bool,
+	/// Parses decorators on items
 	pub decorators: bool,
 	pub generator_keyword: bool,
+	/// Skip **all** comments from the AST
 	pub include_comments: bool,
-
+	/// See [crate::extensions::is_expression::IsExpression]
 	pub is_expressions: bool,
-	pub server_blocks: bool,
-	pub module_blocks: bool,
+	/// Allows functions to be prefixed with 'server'
+	pub server_functions: bool,
+	/// Allows functions to be prefixed with 'module'
+	pub module_functions: bool,
 	/// For LSP allows incomplete AST for completions. TODO tidy up
 	pub slots: bool,
 }
+
 impl ParseOptions {
 	fn get_lex_options(&self) -> LexerOptions {
 		LexerOptions {
@@ -101,6 +107,20 @@ impl ParseOptions {
 			allow_unsupported_characters_in_jsx_attribute_keys: self.special_jsx_attributes,
 		}
 	}
+
+	pub fn all_features() -> Self {
+		Self {
+			jsx: true,
+			special_jsx_attributes: true,
+			include_comments: true,
+			decorators: true,
+			slots: true,
+			generator_keyword: true,
+			server_functions: true,
+			module_functions: true,
+			is_expressions: true,
+		}
+	}
 }
 
 // TODO not sure about some of these defaults, may change in future
@@ -113,8 +133,8 @@ impl Default for ParseOptions {
 			decorators: true,
 			slots: false,
 			generator_keyword: true,
-			server_blocks: false,
-			module_blocks: false,
+			server_functions: false,
+			module_functions: false,
 			is_expressions: true,
 		}
 	}

parser/src/tokens.rs

@@ -322,9 +322,9 @@ pub enum TSXKeyword {
     Private, Public, Protected,
     // TS Keywords
     As, Declare, Readonly, Infer, Is, Satisfies, Namespace, KeyOf,
-    // Extra blocks
+    // Extra function modifiers
     #[cfg(feature = "extras")] Server, #[cfg(feature = "extras")] Module,
-    // Type changes
+    // Type declaration changes
     #[cfg(feature = "extras")] Nominal, #[cfg(feature = "extras")] Performs,
 
     #[cfg(feature = "extras")]

src/ast_explorer.rs

@@ -124,7 +124,12 @@ impl ExplorerSubCommand {
 				let mut fs =
 					parser::source_map::MapFileStore::<parser::source_map::NoPathMap>::default();
 				let source_id = fs.new_source_id(path.unwrap_or_default(), input.clone());
-				let res = Expression::from_string(input, Default::default(), source_id, None);
+				let res = Expression::from_string(
+					input,
+					parser::ParseOptions::all_features(),
+					source_id,
+					None,
+				);
 				match res {
 					Ok(res) => {
 						if cfg.json {
@@ -144,7 +149,12 @@ impl ExplorerSubCommand {
 				let mut fs =
 					parser::source_map::MapFileStore::<parser::source_map::NoPathMap>::default();
 				let source_id = fs.new_source_id(path.unwrap_or_default(), input.clone());
-				let res = Module::from_string(input, Default::default(), source_id, None);
+				let res = Module::from_string(
+					input,
+					parser::ParseOptions::all_features(),
+					source_id,
+					None,
+				);
 				match res {
 					Ok(res) => {
 						if cfg.json {

src/js-cli-and-library/package.json

@@ -13,6 +13,9 @@
 		},
 		"./initialised": {
 			"import": "./dist/initialised.mjs"
+		},
+		"./cli": {
+			"import": "./dist/cli.mjs"
 		}
 	},
 	"scripts": {
@@ -72,4 +75,4 @@
 	"devDependencies": {
 		"unbuild": "^1.1.2"
 	}
-}
+}

src/js-cli-and-library/src/index.mjs

@@ -1,2 +1,2 @@
-import init, { initSync, build, check, parse_expression, parse_module, just_imports } from "../build/ezno_lib.js";
-export { init, initSync, build, check, parse_expression, parse_module, just_imports };
+import init, { initSync, build, check, parse_expression, parse_module, just_imports, get_version } from "../build/ezno_lib.js";
+export { init, initSync, build, check, parse_expression, parse_module, just_imports, get_version };

src/js-cli-and-library/src/initialised.js

@@ -1,4 +1,4 @@
-import { initSync, build, check, parse_expression, parse_module, just_imports } from "./index.mjs";
+import { initSync, build, check, parse_expression, parse_module, just_imports, get_version } from "./index.mjs";
 import { readFileSync } from "node:fs";
 
 const wasmPath = new URL("./shared/ezno_lib_bg.wasm", import.meta.url);
@@ -8,4 +8,4 @@ if (wasmPath.protocol === "https:") {
     initSync(readFileSync(wasmPath));
 }
 
-export { build, check, parse_expression, parse_module, just_imports }
+export { build, check, parse_expression, parse_module, just_imports, get_version }