-
Notifications
You must be signed in to change notification settings - Fork 0
/
scangen.py
executable file
·108 lines (95 loc) · 3.07 KB
/
scangen.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/usr/bin/python3
from argparse import ArgumentParser
from bootstrap import parse, ParseTree, AstLiteral, DFA
def parse_bnf(fn, tbl = {}):
for p in parse(fn):
if p.name in tbl:
raise Exception('%s multiply defind'%p.name)
tbl[p.name] = p
return tbl
def builtin_productions(tbl = {}):
d = {
'__lf__': '\\n',
'__cr__': '\\r',
'__tab__': '\\t',
'__space__': ' ',
'__hash__': '#',
'__lt__': '<',
'__gt__': '>',
'__lsq__': '[',
'__rsq__': ']',
'__lbr__': '{',
'__rbr__': '}',
'__vbar__': '|',
}
for k, v in list(d.items()):
p = ParseTree(k)
p.root = AstLiteral(v)
tbl[p.name] = p
return tbl
def resolve_ambiguity(dfa):
nf = {}
while dfa.final:
i, f = dfa.final.popitem()
if len(f) <= 1:
nf[i] = f
continue
x = sorted([(x.lineno, x) for x in f])
f = ', '.join(sorted(map(lambda x:x.rule_name, f)))
print('Ambiguity: %s, picked %s'%(f, x[0][1].rule_name))
#print x[0][1]
nf[i] = [x[0][1]]
dfa.final = nf
if __name__ == '__main__':
opts = ArgumentParser(description='Generate a tokenizer')
opts.add_argument('production',
metavar='production', type=str,
help = 'Name of the root token rule')
opts.add_argument('files', metavar='file', type=str, nargs='+',
help = 'BNF file')
opts.add_argument('--includedir',
metavar = 'dir',
type = str,
default = '.',
help = 'Directory to place header file')
opts.add_argument('--srcdir',
metavar = 'dir',
type = str,
default = '.',
help = 'Directory to place C file')
opts.add_argument('--dump-graph',
action = 'store_true',
default = False,
help = 'Dump dotty graphs')
opts.add_argument('--no-optimize',
action = 'store_true',
default = False,
help = 'Disable DFA optimization')
opts.add_argument('--base-name',
metavar = 'basename',
default = 'lex',
type = str,
help = 'Set the output filename')
opts.add_argument('--language',
metavar = 'output-language',
default = 'C',
type = str,
help = 'Set the output language')
args = opts.parse_args()
tbl = {}
builtin_productions(tbl)
list(map(lambda x:parse_bnf(x, tbl), args.files))
dfa = DFA(tbl[args.production], tbl)
del tbl
if args.dump_graph:
dfa.dump_graph('dfa.dot')
if not args.no_optimize:
dfa.optimize()
resolve_ambiguity(dfa)
if not args.no_optimize and args.dump_graph:
dfa.dump_graph('optimized.dot')
dfa.write(base_name = args.base_name,
srcdir = args.srcdir,
includedir = args.includedir,
language = args.language)
raise SystemExit(0)