perf: Improve statement parsing performance (#201)

eclipse-che4z · Nov 16, 2021 · c949784 · c949784
1 parent 2265bc4
commit c949784
Show file tree

Hide file tree

Showing 11 changed files with 258 additions and 144 deletions.
diff --git a/parser_library/src/lexing/token_stream.cpp b/parser_library/src/lexing/token_stream.cpp
@@ -20,18 +20,13 @@ using namespace antlr4;
 token_stream::token_stream(antlr4::TokenSource* token_source)
     : antlr4::BufferedTokenStream(token_source)
     , enabled_cont_(false)
-    , enabled_hidden_(false)
     , needSetup_(true)
 {}
 
 void token_stream::enable_continuation() { enabled_cont_ = true; }
 
 void token_stream::disable_continuation() { enabled_cont_ = false; }
 
-void token_stream::enable_hidden() { enabled_hidden_ = true; }
-
-void token_stream::disable_hidden() { enabled_hidden_ = false; }
-
 void token_stream::reset()
 {
     _tokens.clear();
@@ -215,6 +210,5 @@ size_t token_stream::previous_token_on_channel(size_t i)
 bool token_stream::is_on_channel(antlr4::Token* token)
 {
     return token->getChannel() == lexer::Channels::DEFAULT_CHANNEL
-        || (enabled_hidden_ && token->getChannel() == lexer::Channels::HIDDEN_CHANNEL)
         || (enabled_cont_ && token->getType() == lexer::CONTINUATION) || token->getType() == antlr4::Token::EOF;
 }
diff --git a/parser_library/src/lexing/token_stream.h b/parser_library/src/lexing/token_stream.h
@@ -30,7 +30,6 @@ namespace hlasm_plugin::parser_library::lexing {
 class token_stream : public antlr4::BufferedTokenStream
 {
     bool enabled_cont_;
-    bool enabled_hidden_;
     bool needSetup_;
 
 public:
@@ -41,11 +40,6 @@ class token_stream : public antlr4::BufferedTokenStream
     // filter continuation token from the token stream
     void disable_continuation();
 
-    // enable hidden channel tokens in the token stream
-    void enable_hidden();
-    // disable hidden token channel
-    void disable_hidden();
-
     antlr4::Token* LT(ssize_t k) override;
 
     std::string getText(const antlr4::misc::Interval& interval) override;

diff --git a/parser_library/src/parsing/grammar/ca_expr_rules.g4 b/parser_library/src/parsing/grammar/ca_expr_rules.g4
@@ -75,7 +75,7 @@ term returns [ca_expr_ptr ca_expr]
 		auto r = provider.get_range($var_symbol.ctx);
 		$ca_expr = std::make_unique<ca_var_sym>(std::move($var_symbol.vs), r);
 	}
-	| ca_string
+	| {allow_ca_string()}? ca_string
 	{ 
 		auto r = provider.get_range($ca_string.ctx);
 		collector.add_hl_symbol(token_info(r, hl_scopes::string));

diff --git a/parser_library/src/parsing/grammar/deferred_operand_rules.g4 b/parser_library/src/parsing/grammar/deferred_operand_rules.g4
@@ -15,30 +15,109 @@
  //rules for deferred operand
 parser grammar deferred_operand_rules;
 
-deferred_entry returns [vs_ptr vs]
-	: mac_preproc_c					{$vs = std::move($mac_preproc_c.vs);}
-	| apostrophe
+deferred_entry returns [std::vector<vs_ptr> vs]
+	: asterisk
+	| minus
+	| plus
+	| LT
+	| GT
+	| slash
+	| equals
+	| VERTICAL
+	| IDENTIFIER											{collector.add_hl_symbol(token_info(provider.get_range($IDENTIFIER), hl_scopes::operand));}
+	| NUM													{collector.add_hl_symbol(token_info(provider.get_range($NUM), hl_scopes::operand));}
+	| ORDSYMBOL												{collector.add_hl_symbol(token_info(provider.get_range($ORDSYMBOL), hl_scopes::operand));}
+	| dot
+	| lpar
+	| rpar
+	| attr
+	|
+	ap1=APOSTROPHE
+	{disable_ca_string();}
+	(
+		(APOSTROPHE|ATTR) (APOSTROPHE|ATTR)
+		|
+		(
+			{std::string name;}
+			AMPERSAND (ORDSYMBOL {name += $ORDSYMBOL->getText();})+
+			{
+				auto r = provider.get_range($AMPERSAND,$ORDSYMBOL);
+				$vs.push_back(std::make_unique<basic_variable_symbol>(hlasm_ctx->ids().add(std::move(name)), std::vector<ca_expr_ptr>(), r));
+				collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol));
+			}
+		)
+		|
+		l_sp_ch_v
+	)*
+	{enable_ca_string();}
+	ap2=(APOSTROPHE|ATTR)
+	{
+		collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string));
+	}
 	| comma
 	| AMPERSAND
-	| IGNORED; 
+	(
+		ORDSYMBOL
+		{
+			auto name = $ORDSYMBOL->getText();
+		}
+		(
+			ORDSYMBOL
+			{
+				name += $ORDSYMBOL->getText();
+			}
+		)*
+		{
+			auto r = provider.get_range($AMPERSAND,$ORDSYMBOL);
+			$vs.push_back(std::make_unique<basic_variable_symbol>(hlasm_ctx->ids().add(std::move(name)), std::vector<ca_expr_ptr>(), r));
+			collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol));
+		}
+		|
+		LPAR
+		|
+		AMPERSAND
+	)?
+	;
+	finally
+	{enable_ca_string();}
 
 def_string_body
 	: string_ch_v
 	| IGNORED
 	| CONTINUATION;
 
 def_string returns [concat_chain chain]
-	: ap1=APOSTROPHE def_string_body* ap2=(APOSTROPHE|ATTR)	
+	: ap1=APOSTROPHE def_string_body*? ap2=(APOSTROPHE|ATTR)
 	{ 
 		collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string)); 
 	};
 	finally
 	{concatenation_point::clear_concat_chain($chain);}
 
 deferred_op_rem returns [remark_list remarks, std::vector<vs_ptr> var_list]
-	: (deferred_entry {if ($deferred_entry.vs) $var_list.push_back(std::move($deferred_entry.vs));})* 
-	remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);} 
-	(CONTINUATION 
-	(deferred_entry {if ($deferred_entry.vs) $var_list.push_back(std::move($deferred_entry.vs));})* 
-	remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);} 
-	)* IGNORED*;
+	:
+	(
+		deferred_entry
+		{
+			for (auto&v : $deferred_entry.vs)
+				$var_list.push_back(std::move(v));
+		}
+	)*
+	{enable_continuation();}
+	remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);}
+	(
+		CONTINUATION
+		{disable_continuation();}
+		(
+			deferred_entry
+			{
+				for (auto&v : $deferred_entry.vs)
+					$var_list.push_back(std::move(v));
+			}
+		)*
+		{enable_continuation();}
+		remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);}
+	)*
+	;
+	finally
+	{disable_continuation();}
diff --git a/parser_library/src/parsing/grammar/hlasmparser.g4 b/parser_library/src/parsing/grammar/hlasmparser.g4
@@ -162,15 +162,16 @@ first_part
 	};
 
 operand_field_rest
-	: .*?;
+	: (~EOF)*;
 
 lab_instr returns [std::optional<std::string> op_text, range op_range]
-	: first_part {enable_hidden();} operand_field_rest {disable_hidden();} EOF
+	: first_part operand_field_rest EOF
 	{
 		if (!$first_part.ctx->exception)
 		{
-			$op_text = $operand_field_rest.ctx->getText();
-			$op_range = provider.get_range($operand_field_rest.ctx);
+			auto op_index = $first_part.stop->getTokenIndex()+1;
+			$op_text = _input->getText(misc::Interval(op_index,_input->size()-1));
+			$op_range = provider.get_range(_input->get(op_index),_input->get(_input->size()-1));
 		}
 	}
 	| SPACE? EOF

diff --git a/parser_library/src/parsing/grammar/macro_operand_rules.g4 b/parser_library/src/parsing/grammar/macro_operand_rules.g4
@@ -51,10 +51,19 @@ mac_preproc_c returns [vs_ptr vs]
 	| NUM													{collector.add_hl_symbol(token_info(provider.get_range($NUM), hl_scopes::operand));}
 	| ORDSYMBOL												{collector.add_hl_symbol(token_info(provider.get_range($ORDSYMBOL), hl_scopes::operand));}
 	| dot									
-	| AMPERSAND ORDSYMBOL									
+	| AMPERSAND ORDSYMBOL
+	{
+		auto name = $ORDSYMBOL->getText();
+	}
+	(
+		CONTINUATION IGNORED* ORDSYMBOL
+		{
+			name += $ORDSYMBOL->getText();
+		}
+	)*
 	{
 		auto r = provider.get_range($AMPERSAND,$ORDSYMBOL);
-		$vs = std::make_unique<basic_variable_symbol>(hlasm_ctx->ids().add($ORDSYMBOL->getText()), std::vector<ca_expr_ptr>(), r);
+		$vs = std::make_unique<basic_variable_symbol>(hlasm_ctx->ids().add(std::move(name)), std::vector<ca_expr_ptr>(), r);
 		collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol));
 	}
 	| AMPERSAND LPAR