diff --git a/parser_library/src/lexing/token_stream.cpp b/parser_library/src/lexing/token_stream.cpp index dd097b64c..ef5b587d2 100644 --- a/parser_library/src/lexing/token_stream.cpp +++ b/parser_library/src/lexing/token_stream.cpp @@ -20,7 +20,6 @@ using namespace antlr4; token_stream::token_stream(antlr4::TokenSource* token_source) : antlr4::BufferedTokenStream(token_source) , enabled_cont_(false) - , enabled_hidden_(false) , needSetup_(true) {} @@ -28,10 +27,6 @@ void token_stream::enable_continuation() { enabled_cont_ = true; } void token_stream::disable_continuation() { enabled_cont_ = false; } -void token_stream::enable_hidden() { enabled_hidden_ = true; } - -void token_stream::disable_hidden() { enabled_hidden_ = false; } - void token_stream::reset() { _tokens.clear(); @@ -215,6 +210,5 @@ size_t token_stream::previous_token_on_channel(size_t i) bool token_stream::is_on_channel(antlr4::Token* token) { return token->getChannel() == lexer::Channels::DEFAULT_CHANNEL - || (enabled_hidden_ && token->getChannel() == lexer::Channels::HIDDEN_CHANNEL) || (enabled_cont_ && token->getType() == lexer::CONTINUATION) || token->getType() == antlr4::Token::EOF; } diff --git a/parser_library/src/lexing/token_stream.h b/parser_library/src/lexing/token_stream.h index 531ee5a5d..1d1edf231 100644 --- a/parser_library/src/lexing/token_stream.h +++ b/parser_library/src/lexing/token_stream.h @@ -30,7 +30,6 @@ namespace hlasm_plugin::parser_library::lexing { class token_stream : public antlr4::BufferedTokenStream { bool enabled_cont_; - bool enabled_hidden_; bool needSetup_; public: @@ -41,11 +40,6 @@ class token_stream : public antlr4::BufferedTokenStream // filter continuation token from the token stream void disable_continuation(); - // enable hidden channel tokens in the token stream - void enable_hidden(); - // disable hidden token channel - void disable_hidden(); - antlr4::Token* LT(ssize_t k) override; std::string getText(const antlr4::misc::Interval& interval) override; diff --git a/parser_library/src/parsing/grammar/ca_expr_rules.g4 b/parser_library/src/parsing/grammar/ca_expr_rules.g4 index 320001f09..da54b4592 100644 --- a/parser_library/src/parsing/grammar/ca_expr_rules.g4 +++ b/parser_library/src/parsing/grammar/ca_expr_rules.g4 @@ -75,7 +75,7 @@ term returns [ca_expr_ptr ca_expr] auto r = provider.get_range($var_symbol.ctx); $ca_expr = std::make_unique(std::move($var_symbol.vs), r); } - | ca_string + | {allow_ca_string()}? ca_string { auto r = provider.get_range($ca_string.ctx); collector.add_hl_symbol(token_info(r, hl_scopes::string)); diff --git a/parser_library/src/parsing/grammar/deferred_operand_rules.g4 b/parser_library/src/parsing/grammar/deferred_operand_rules.g4 index f4510dd75..fa826fa86 100644 --- a/parser_library/src/parsing/grammar/deferred_operand_rules.g4 +++ b/parser_library/src/parsing/grammar/deferred_operand_rules.g4 @@ -15,12 +15,71 @@ //rules for deferred operand parser grammar deferred_operand_rules; -deferred_entry returns [vs_ptr vs] - : mac_preproc_c {$vs = std::move($mac_preproc_c.vs);} - | apostrophe +deferred_entry returns [std::vector vs] + : asterisk + | minus + | plus + | LT + | GT + | slash + | equals + | VERTICAL + | IDENTIFIER {collector.add_hl_symbol(token_info(provider.get_range($IDENTIFIER), hl_scopes::operand));} + | NUM {collector.add_hl_symbol(token_info(provider.get_range($NUM), hl_scopes::operand));} + | ORDSYMBOL {collector.add_hl_symbol(token_info(provider.get_range($ORDSYMBOL), hl_scopes::operand));} + | dot + | lpar + | rpar + | attr + | + ap1=APOSTROPHE + {disable_ca_string();} + ( + (APOSTROPHE|ATTR) (APOSTROPHE|ATTR) + | + ( + {std::string name;} + AMPERSAND (ORDSYMBOL {name += $ORDSYMBOL->getText();})+ + { + auto r = provider.get_range($AMPERSAND,$ORDSYMBOL); + $vs.push_back(std::make_unique(hlasm_ctx->ids().add(std::move(name)), std::vector(), r)); + collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol)); + } + ) + | + l_sp_ch_v + )* + {enable_ca_string();} + ap2=(APOSTROPHE|ATTR) + { + collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string)); + } | comma | AMPERSAND - | IGNORED; + ( + ORDSYMBOL + { + auto name = $ORDSYMBOL->getText(); + } + ( + ORDSYMBOL + { + name += $ORDSYMBOL->getText(); + } + )* + { + auto r = provider.get_range($AMPERSAND,$ORDSYMBOL); + $vs.push_back(std::make_unique(hlasm_ctx->ids().add(std::move(name)), std::vector(), r)); + collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol)); + } + | + LPAR + | + AMPERSAND + )? + ; + finally + {enable_ca_string();} def_string_body : string_ch_v @@ -28,7 +87,7 @@ def_string_body | CONTINUATION; def_string returns [concat_chain chain] - : ap1=APOSTROPHE def_string_body* ap2=(APOSTROPHE|ATTR) + : ap1=APOSTROPHE def_string_body*? ap2=(APOSTROPHE|ATTR) { collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string)); }; @@ -36,9 +95,29 @@ def_string returns [concat_chain chain] {concatenation_point::clear_concat_chain($chain);} deferred_op_rem returns [remark_list remarks, std::vector var_list] - : (deferred_entry {if ($deferred_entry.vs) $var_list.push_back(std::move($deferred_entry.vs));})* - remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);} - (CONTINUATION - (deferred_entry {if ($deferred_entry.vs) $var_list.push_back(std::move($deferred_entry.vs));})* - remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);} - )* IGNORED*; \ No newline at end of file + : + ( + deferred_entry + { + for (auto&v : $deferred_entry.vs) + $var_list.push_back(std::move(v)); + } + )* + {enable_continuation();} + remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);} + ( + CONTINUATION + {disable_continuation();} + ( + deferred_entry + { + for (auto&v : $deferred_entry.vs) + $var_list.push_back(std::move(v)); + } + )* + {enable_continuation();} + remark_o {if($remark_o.value) $remarks.push_back(*$remark_o.value);} + )* + ; + finally + {disable_continuation();} \ No newline at end of file diff --git a/parser_library/src/parsing/grammar/hlasmparser.g4 b/parser_library/src/parsing/grammar/hlasmparser.g4 index 7e6cfe635..ceeb32b72 100644 --- a/parser_library/src/parsing/grammar/hlasmparser.g4 +++ b/parser_library/src/parsing/grammar/hlasmparser.g4 @@ -162,15 +162,16 @@ first_part }; operand_field_rest - : .*?; + : (~EOF)*; lab_instr returns [std::optional op_text, range op_range] - : first_part {enable_hidden();} operand_field_rest {disable_hidden();} EOF + : first_part operand_field_rest EOF { if (!$first_part.ctx->exception) { - $op_text = $operand_field_rest.ctx->getText(); - $op_range = provider.get_range($operand_field_rest.ctx); + auto op_index = $first_part.stop->getTokenIndex()+1; + $op_text = _input->getText(misc::Interval(op_index,_input->size()-1)); + $op_range = provider.get_range(_input->get(op_index),_input->get(_input->size()-1)); } } | SPACE? EOF diff --git a/parser_library/src/parsing/grammar/macro_operand_rules.g4 b/parser_library/src/parsing/grammar/macro_operand_rules.g4 index 902653a1d..f80a9cc84 100644 --- a/parser_library/src/parsing/grammar/macro_operand_rules.g4 +++ b/parser_library/src/parsing/grammar/macro_operand_rules.g4 @@ -51,10 +51,19 @@ mac_preproc_c returns [vs_ptr vs] | NUM {collector.add_hl_symbol(token_info(provider.get_range($NUM), hl_scopes::operand));} | ORDSYMBOL {collector.add_hl_symbol(token_info(provider.get_range($ORDSYMBOL), hl_scopes::operand));} | dot - | AMPERSAND ORDSYMBOL + | AMPERSAND ORDSYMBOL + { + auto name = $ORDSYMBOL->getText(); + } + ( + CONTINUATION IGNORED* ORDSYMBOL + { + name += $ORDSYMBOL->getText(); + } + )* { auto r = provider.get_range($AMPERSAND,$ORDSYMBOL); - $vs = std::make_unique(hlasm_ctx->ids().add($ORDSYMBOL->getText()), std::vector(), r); + $vs = std::make_unique(hlasm_ctx->ids().add(std::move(name)), std::vector(), r); collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol)); } | AMPERSAND LPAR diff --git a/parser_library/src/parsing/grammar/operand_field_rules.g4 b/parser_library/src/parsing/grammar/operand_field_rules.g4 index 3694b2f71..31c8811b0 100644 --- a/parser_library/src/parsing/grammar/operand_field_rules.g4 +++ b/parser_library/src/parsing/grammar/operand_field_rules.g4 @@ -152,7 +152,10 @@ operand_asm returns [operand_ptr op] //////////////////////////////////////// ca op_rem_body_ca - : SPACE+ op_rem_body_alt_ca + : + SPACE* EOF {collector.set_operand_remark_field(provider.get_range(_localctx));} + | + SPACE+ op_rem_body_alt_ca { auto line_range = provider.get_range($op_rem_body_alt_ca.ctx); collector.set_operand_remark_field(std::move($op_rem_body_alt_ca.line.operands), std::move($op_rem_body_alt_ca.line.remarks), line_range); @@ -162,64 +165,48 @@ op_rem_body_ca auto remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; auto line_range = provider.get_range($remark_o.ctx); collector.set_operand_remark_field(operand_list(), std::move(remarks), line_range); - } EOF - | {collector.set_operand_remark_field(provider.get_range(_localctx));} EOF; + } EOF; op_rem_body_alt_ca returns [op_rem line] - : alt_op_list_comma_ca cont_ca - { - $line.operands = std::move($alt_op_list_comma_ca.operands); - $line.operands.insert($line.operands.end(), std::make_move_iterator($cont_ca.line.operands.begin()), std::make_move_iterator($cont_ca.line.operands.end())); - $line.remarks = std::move($cont_ca.line.remarks); - } - | alt_op_list_comma_o_ca ca_op remark_o - { - $alt_op_list_comma_o_ca.operands.push_back(std::move($ca_op.op)); - $line.operands = std::move($alt_op_list_comma_o_ca.operands); - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - }; - - -cont_ca returns [op_rem line] - : {enable_continuation();} cont_body_ca {disable_continuation();} {$line = std::move($cont_body_ca.line);}; - -cont_body_ca returns [op_rem line] - : remark_o - { - auto tmp = std::make_unique(provider.get_empty_range( $remark_o.ctx->getStart())); - $line.operands.push_back(std::move(tmp)); - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - } - | r1=remark_o CONTINUATION {disable_continuation();} /*empty op*/ r2=remark_o - { - if($r1.value) $line.remarks.push_back(*$r1.value); - auto tmp = std::make_unique(range(provider.get_range( $r2.ctx).start)); - $line.operands.push_back(std::move(tmp)); - if($r2.value) $line.remarks.push_back(*$r2.value); - } - | remark_o CONTINUATION {disable_continuation();} next=op_rem_body_alt_ca - { - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - $line.remarks.insert($line.remarks.end(),std::make_move_iterator($next.line.remarks.begin()),std::make_move_iterator($next.line.remarks.end())); - $line.operands = std::move($next.line.operands); - }; - -alt_op_list_comma_o_ca returns [std::vector operands] : - | alt_op_list_comma_ca {$operands = std::move($alt_op_list_comma_ca.operands);}; - -alt_op_list_comma_ca returns [std::vector operands] - : alt_operand_ca comma {$operands.push_back(std::move($alt_operand_ca.op)); } - | tmp=alt_op_list_comma_ca alt_operand_ca comma {$tmp.operands.push_back(std::move($alt_operand_ca.op)); $operands = std::move($tmp.operands); }; - -alt_operand_ca returns [operand_ptr op] - : ca_op {$op = std::move($ca_op.op);} - | {$op = std::make_unique(provider.get_empty_range( _localctx->getStart()));}; + ( + ( + ca_op? comma + { + if ($ca_op.ctx && $ca_op.op) + $line.operands.push_back(std::move($ca_op.op)); + else + $line.operands.push_back(std::make_unique(provider.get_empty_range($comma.ctx->getStart()))); + } + )+ + {enable_continuation();} + ( + r1=remark_o CONTINUATION + { + if($r1.value) $line.remarks.push_back(std::move(*$r1.value)); + } + )? + {disable_continuation();} + )* + ( + last_ca_op=ca_op? last_remark=remark_o + { + if ($last_ca_op.ctx) + $line.operands.push_back(std::move($last_ca_op.op)); + else + $line.operands.push_back(std::make_unique(provider.get_empty_range($remark_o.ctx->getStart()))); + if ($last_remark.value) + $line.remarks.push_back(std::move(*$last_remark.value)); + } + ); //////////////////////////////////////// mac op_rem_body_mac returns [op_rem line, range line_range] - : SPACE+ op_rem_body_alt_mac + : + SPACE* EOF {$line_range = provider.get_range(_localctx);} + | + SPACE+ op_rem_body_alt_mac { $line = std::move($op_rem_body_alt_mac.line); $line_range = provider.get_range($op_rem_body_alt_mac.ctx); @@ -228,59 +215,38 @@ op_rem_body_mac returns [op_rem line, range line_range] { $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; $line_range = provider.get_range($remark_o.ctx); - } EOF - | {$line_range = provider.get_range(_localctx);} EOF; + } EOF; op_rem_body_alt_mac returns [op_rem line] - : alt_op_list_comma_mac cont_mac - { - $line.operands = std::move($alt_op_list_comma_mac.operands); - $line.operands.insert($line.operands.end(), std::make_move_iterator($cont_mac.line.operands.begin()), std::make_move_iterator($cont_mac.line.operands.end())); - $line.remarks = std::move($cont_mac.line.remarks); - } - | alt_op_list_comma_o_mac mac_op remark_o - { - $alt_op_list_comma_o_mac.operands.push_back(std::move($mac_op.op)); - $line.operands = std::move($alt_op_list_comma_o_mac.operands); - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - }; - - -cont_mac returns [op_rem line] - : {enable_continuation();} cont_body_mac {disable_continuation();} {$line = std::move($cont_body_mac.line);}; - -cont_body_mac returns [op_rem line] - : remark_o - { - auto tmp = std::make_unique(provider.get_empty_range( $remark_o.ctx->getStart())); - $line.operands.push_back(std::move(tmp)); - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - } - | r1=remark_o CONTINUATION {disable_continuation();} /*empty op*/ r2=remark_o - { - if($r1.value) $line.remarks.push_back(*$r1.value); - auto tmp = std::make_unique(range(provider.get_range( $r2.ctx).start)); - $line.operands.push_back(std::move(tmp)); - if($r2.value) $line.remarks.push_back(*$r2.value); - } - | remark_o CONTINUATION {disable_continuation();} next=op_rem_body_alt_mac - { - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - $line.remarks.insert($line.remarks.end(),std::make_move_iterator($next.line.remarks.begin()),std::make_move_iterator($next.line.remarks.end())); - $line.operands = std::move($next.line.operands); - }; - -alt_op_list_comma_o_mac returns [std::vector operands] : - | alt_op_list_comma_mac {$operands = std::move($alt_op_list_comma_mac.operands);}; - -alt_op_list_comma_mac returns [std::vector operands] - : alt_operand_mac comma {$operands.push_back(std::move($alt_operand_mac.op)); } - | tmp=alt_op_list_comma_mac alt_operand_mac comma {$tmp.operands.push_back(std::move($alt_operand_mac.op)); $operands = std::move($tmp.operands); }; - -alt_operand_mac returns [operand_ptr op] - : mac_op {$op = std::move($mac_op.op);} - | {$op = std::make_unique(provider.get_empty_range( _localctx->getStart()));}; + ( + ( + mac_op? comma + { + if ($mac_op.ctx && $mac_op.op) + $line.operands.push_back(std::move($mac_op.op)); + else + $line.operands.push_back(std::make_unique(provider.get_empty_range($comma.ctx->getStart()))); + } + )+ + {enable_continuation();} + ( + r1=remark_o CONTINUATION + { + if($r1.value) $line.remarks.push_back(std::move(*$r1.value)); + } + )? + {disable_continuation();} + )* + ( + last_mac_op=mac_op? last_remark=remark_o + { + if ($last_mac_op.ctx) + $line.operands.push_back(std::move($last_mac_op.op)); + if ($last_remark.value) + $line.remarks.push_back(std::move(*$last_remark.value)); + } + ); ///////////// @@ -292,10 +258,10 @@ op_rem_body_deferred { collector.set_operand_remark_field(provider.get_empty_range(_localctx->getStart())); } EOF - | SPACE {enable_hidden();} deferred_op_rem {disable_hidden();} + | SPACE deferred_op_rem EOF { - auto r = provider.get_range( $deferred_op_rem.ctx); - collector.set_operand_remark_field($deferred_op_rem.ctx->getText(),std::move($deferred_op_rem.var_list),std::move($deferred_op_rem.remarks),r); + auto r = provider.get_range(_input->get($SPACE.index+1),_input->get(_input->size()-1)); + collector.set_operand_remark_field(_input->getText(misc::Interval($SPACE.index+1,_input->size()-1)),std::move($deferred_op_rem.var_list),std::move($deferred_op_rem.remarks),r); } EOF; op_rem_body_noop @@ -367,15 +333,17 @@ op_rem_body_asm_r returns [op_rem line] //////////////////////////////////////// mac_r op_rem_body_mac_r returns [op_rem line] - : op_rem_body_alt_mac + : + SPACE* EOF + | + op_rem_body_alt_mac { $line = std::move($op_rem_body_alt_mac.line); } EOF | remark_o { $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - } EOF - | EOF; + } EOF; op_rem_body_noop_r : remark_o EOF; \ No newline at end of file diff --git a/parser_library/src/parsing/parser_impl.cpp b/parser_library/src/parsing/parser_impl.cpp index 9f83a2d76..f297d2e40 100644 --- a/parser_library/src/parsing/parser_impl.cpp +++ b/parser_library/src/parsing/parser_impl.cpp @@ -74,10 +74,6 @@ void parser_impl::enable_continuation() { input.enable_continuation(); } void parser_impl::disable_continuation() { input.disable_continuation(); } -void parser_impl::enable_hidden() { input.enable_hidden(); } - -void parser_impl::disable_hidden() { input.disable_hidden(); } - bool parser_impl::is_self_def() { std::string tmp(_input->LT(1)->getText()); diff --git a/parser_library/src/parsing/parser_impl.h b/parser_library/src/parsing/parser_impl.h index 9b609f90d..4e57960cf 100644 --- a/parser_library/src/parsing/parser_impl.h +++ b/parser_library/src/parsing/parser_impl.h @@ -64,11 +64,14 @@ class parser_impl : public antlr4::Parser protected: void enable_continuation(); void disable_continuation(); - void enable_hidden(); - void disable_hidden(); bool is_self_def(); bool is_data_attr(); bool is_var_def(); + + bool allow_ca_string() const { return ca_string_enabled; } + void enable_ca_string() { ca_string_enabled = true; } + void disable_ca_string() { ca_string_enabled = false; } + self_def_t parse_self_def_term(const std::string& option, const std::string& value, range term_range); context::data_attr_kind get_attribute(std::string attr_data); context::id_index parse_identifier(std::string value, range id_range); @@ -92,6 +95,8 @@ class parser_impl : public antlr4::Parser antlr4::misc::IntervalSet getExpectedTokens() override; diagnostic_op_consumer* diagnoser_ = nullptr; parser_error_listener err_listener_; + + bool ca_string_enabled = true; }; // structure containing parser components diff --git a/parser_library/test/parsing/CMakeLists.txt b/parser_library/test/parsing/CMakeLists.txt index 7964b4575..bf033bd1f 100644 --- a/parser_library/test/parsing/CMakeLists.txt +++ b/parser_library/test/parsing/CMakeLists.txt @@ -11,6 +11,7 @@ # Broadcom, Inc. - initial API and implementation target_sources(library_test PRIVATE + deferred_statement_test.cpp parser_model_test.cpp parser_range_test.cpp parser_test.cpp diff --git a/parser_library/test/parsing/deferred_statement_test.cpp b/parser_library/test/parsing/deferred_statement_test.cpp new file mode 100644 index 000000000..75a474d79 --- /dev/null +++ b/parser_library/test/parsing/deferred_statement_test.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#include "gtest/gtest.h" + +#include "../common_testing.h" + + +TEST(deferred_statement, split_var) +{ + std::string input = R"( + MACRO + MAC &AA + &AA C' &AX + A' + MEND + + MAC DC +)"; + analyzer a(input); + a.analyze(); + + a.collect_diags(); + EXPECT_TRUE(a.diags().empty()); + + auto aa1 = a.context().lsp_ctx->hover("", { 3, 71 }); + auto aa2 = a.context().lsp_ctx->hover("", { 4, 16 }); + EXPECT_EQ(aa1, "MACRO parameter"); + EXPECT_EQ(aa2, "MACRO parameter"); +} + +TEST(deferred_statement, nested_macro_def_with_continuation) +{ + std::string input = R"( + MACRO + MAC + + MACRO + MAC2 X + &OP=, X + &OP2= + + MEND + + MEND + + MAC + + MAC2 OP=A,OP2=B +)"; + analyzer a(input); + a.analyze(); + + a.collect_diags(); + EXPECT_TRUE(a.diags().empty()); +} \ No newline at end of file