-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokens.py
39 lines (34 loc) · 1.05 KB
/
tokens.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
from position import Position
DIGITS = "0123456789"
# token type => TT
TT_INT = "INT"
TT_FLOAT = "FLOAT"
TT_PLUS = "PLUS"
TT_MINUS = "MINUS"
TT_MUL = "MUL"
TT_DIV = "DIV"
TT_LPAREN = "LPAREN"
TT_RPAREN = "RPAREN"
TT_EOF = "EOF" # 终止符
class Token(object):
# <token-name, attribute-value >
def __init__(self, ttype: str, value=None, pos_start: Position = None, pos_end: Position = None):
"""
:param ttype: 类型
:param value:
:param pos_start: Position类实例,当前起始位置
:param pos_end: Position类实例,当前结束位置
"""
self.ttype = ttype
self.value = value
if pos_start:
# Token 单个字符, + => pos_start = pos_end, advance
self.pos_start = pos_start.copy()
self.pos_end = pos_start.copy()
self.pos_end.advance(self.value)
if pos_end:
self.pos_end = pos_end.copy()
def __repr__(self):
if self.value:
return f'{self.ttype}: {self.value}'
return f'{self.ttype}'