From 1a8e8dfde5fcbbae422627f1ff8a8fa89ed179f4 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Tue, 30 Oct 2018 14:46:58 +1100 Subject: [PATCH] adding tests, also making find_tokens not use string algebra but replace with nextind/prevind --- src/parser/find_tokens.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parser/find_tokens.jl b/src/parser/find_tokens.jl index b10570286..59ed95a2a 100644 --- a/src/parser/find_tokens.jl +++ b/src/parser/find_tokens.jl @@ -48,20 +48,20 @@ function find_tokens(str::AbstractString, tokens_dict::Dict, stack = subs(str, head_idx, endchar_idx) if λ(stack) # offset==True --> looked at 1 extra char (lookahead) - head_idx = endchar_idx - offset + head_idx = prevind(str, endchar_idx, offset) push!(tokens, Token(case, chop(stack, tail=offset))) # token identified, no need to check other cases. break end else # rule-based match, greedy catch until fail stack, shift = head, 1 - nextchar_idx = head_idx + shift + nextchar_idx = nextind(str, head_idx) while λ(shift, str[nextchar_idx]) stack = subs(str, head_idx, nextchar_idx) shift += 1 - nextchar_idx += 1 + nextchar_idx = nextind(str, nextchar_idx) end - endchar_idx = nextchar_idx - 1 + endchar_idx = prevind(str, nextchar_idx) if endchar_idx > head_idx push!(tokens, Token(case, stack)) head_idx = endchar_idx