Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use Struct for each token #639

Merged
merged 1 commit into from
Jul 16, 2018
Merged

Use Struct for each token #639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 36 additions & 64 deletions lib/rdoc/parser/ripper_state_lex.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ class RDoc::Parser::RipperStateLex
# TODO: Remove this constants after Ruby 2.4 EOL
RIPPER_HAS_LEX_STATE = Ripper::Filter.method_defined?(:state)

Token = Struct.new(:line_no, :char_no, :kind, :text, :state)

EXPR_NONE = 0
EXPR_BEG = 1
EXPR_END = 2
Expand Down Expand Up @@ -48,7 +50,7 @@ def on_nl(tok, data)
@continue = false
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
end
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_ignored_nl(tok, data)
Expand All @@ -59,7 +61,7 @@ def on_ignored_nl(tok, data)
@continue = false
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
end
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_op(tok, data)
Expand Down Expand Up @@ -101,7 +103,7 @@ def on_op(tok, data)
@lex_state = EXPR_BEG
end
end
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_kw(tok, data)
Expand Down Expand Up @@ -130,54 +132,54 @@ def on_kw(tok, data)
@lex_state = EXPR_END
end
end
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_tstring_beg(tok, data)
@lex_state = EXPR_BEG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_tstring_end(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_CHAR(tok, data)
@lex_state = EXPR_END
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_period(tok, data)
@lex_state = EXPR_DOT
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_int(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_float(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_rational(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_imaginary(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_symbeg(tok, data)
@lex_state = EXPR_FNAME
@continue = true
@in_fname = true
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

private def on_variables(event, tok, data)
Expand All @@ -196,7 +198,7 @@ def on_symbeg(tok, data)
else
@lex_state = EXPR_CMDARG
end
@callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, event, tok, @lex_state))
end

def on_ident(tok, data)
Expand Down Expand Up @@ -225,32 +227,32 @@ def on_backref(tok, data)

def on_lparen(tok, data)
@lex_state = EXPR_LABEL | EXPR_BEG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_rparen(tok, data)
@lex_state = EXPR_ENDFN
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_lbrace(tok, data)
@lex_state = EXPR_LABEL | EXPR_BEG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_rbrace(tok, data)
@lex_state = EXPR_ENDARG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_lbracket(tok, data)
@lex_state = EXPR_LABEL | EXPR_BEG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_rbracket(tok, data)
@lex_state = EXPR_ENDARG
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_const(tok, data)
Expand All @@ -262,36 +264,36 @@ def on_const(tok, data)
else
@lex_state = EXPR_CMDARG
end
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_sp(tok, data)
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_comma(tok, data)
@lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_comment(tok, data)
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_ignored_sp(tok, data)
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
end

def on_heredoc_end(tok, data)
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
@lex_state = EXPR_BEG
end

def on_default(event, tok, data)
reset
@callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
@callback.call(Token.new(lineno, column, event, tok, @lex_state))
end

def each(&block)
Expand All @@ -306,7 +308,7 @@ def initialize(code)
end

def on_default(event, tok, data)
@callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => state})
@callback.call(Token.new(lineno, column, event, tok, state))
end

def each(&block)
Expand Down Expand Up @@ -367,7 +369,7 @@ def get_squashed_tk

private def get_symbol_tk(tk)
is_symbol = true
symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
symbol_tk = Token.new(tk.line_no, tk.char_no, :on_symbol)
if ":'" == tk[:text] or ':"' == tk[:text]
tk1 = get_string_tk(tk)
symbol_tk[:text] = tk1[:text]
Expand Down Expand Up @@ -436,13 +438,7 @@ def get_squashed_tk
end
end
end
{
:line_no => tk[:line_no],
:char_no => tk[:char_no],
:kind => kind,
:text => string,
:state => state
}
Token.new(tk.line_no, tk.char_no, kind, string, state)
end

private def get_regexp_tk(tk)
Expand All @@ -460,13 +456,7 @@ def get_squashed_tk
string = string + inner_str_tk[:text]
end
end
{
:line_no => tk[:line_no],
:char_no => tk[:char_no],
:kind => :on_regexp,
:text => string,
:state => state
}
Token.new(tk.line_no, tk.char_no, :on_regexp, string, state)
end

private def get_embdoc_tk(tk)
Expand All @@ -475,13 +465,7 @@ def get_squashed_tk
string = string + embdoc_tk[:text]
end
string = string + embdoc_tk[:text]
{
:line_no => tk[:line_no],
:char_no => tk[:char_no],
:kind => :on_embdoc,
:text => string,
:state => embdoc_tk[:state]
}
Token.new(tk.line_no, tk.char_no, :on_embdoc, string, embdoc_tk.state)
end

private def get_heredoc_tk(heredoc_name, indent)
Expand All @@ -499,13 +483,7 @@ def get_squashed_tk
start_tk = tk unless start_tk
prev_tk = tk unless prev_tk
@buf.unshift tk # closing heredoc
heredoc_tk = {
:line_no => start_tk[:line_no],
:char_no => start_tk[:char_no],
:kind => :on_heredoc,
:text => string,
:state => prev_tk[:state]
}
heredoc_tk = Token.new(start_tk.line_no, start_tk.char_no, :on_heredoc, string, prev_tk.state)
@buf.unshift heredoc_tk
end

Expand Down Expand Up @@ -561,13 +539,7 @@ def get_squashed_tk
end
end
text = "#{start_token}#{string}#{end_token}"
{
:line_no => line_no,
:char_no => char_no,
:kind => :on_dstring,
:text => text,
:state => state
}
Token.new(line_no, char_no, :on_dstring, text, state)
end

private def get_op_tk(tk)
Expand Down
40 changes: 20 additions & 20 deletions lib/rdoc/parser/ruby.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,17 +536,17 @@ def get_included_module_with_optional_parens
def get_end_token tk # :nodoc:
case tk[:kind]
when :on_lparen
{
:kind => :on_rparen,
:text => ')'
}
token = RDoc::Parser::RipperStateLex::Token.new
token[:kind] = :on_rparen
token[:text] = ')'
token
when :on_rparen
nil
else
{
:kind => :on_nl,
:text => "\n"
}
token = RDoc::Parser::RipperStateLex::Token.new
token[:kind] = :on_nl
token[:text] = "\n"
token
end
end

Expand Down Expand Up @@ -1104,10 +1104,10 @@ def parse_comment_ghost container, text, name, column, line_no, # :nodoc:
record_location meth

meth.start_collecting_tokens
indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
meth.add_tokens [position_comment, newline, indent]

meth.params =
Expand Down Expand Up @@ -1147,10 +1147,10 @@ def parse_comment_tomdoc container, tk, comment
meth.line = line_no

meth.start_collecting_tokens
indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
meth.add_tokens [position_comment, newline, indent]

meth.call_seq = signature
Expand Down Expand Up @@ -1315,10 +1315,10 @@ def parse_meta_method(container, single, tk, comment)
remove_token_listener self

meth.start_collecting_tokens
indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
meth.add_tokens [position_comment, newline, indent]
meth.add_tokens @token_stream

Expand Down Expand Up @@ -1418,10 +1418,10 @@ def parse_method(container, single, tk, comment)
meth.line = line_no

meth.start_collecting_tokens
indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
token = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
token = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
meth.add_tokens [token, newline, indent]
meth.add_tokens @token_stream

Expand Down