diff --git a/internal/pkg/parsing/lexer/lexer.go b/internal/pkg/parsing/lexer/lexer.go index 7894220c4e..1cfd61a36b 100644 --- a/internal/pkg/parsing/lexer/lexer.go +++ b/internal/pkg/parsing/lexer/lexer.go @@ -12,7 +12,7 @@ import ( const ( NoState = -1 NumStates = 328 - NumSymbols = 579 + NumSymbols = 580 ) type Lexer struct { @@ -702,10 +702,11 @@ Lexer symbols: 570: 'A'-'Z' 571: 'a'-'z' 572: '0'-'9' -573: \u0100-\U0010ffff -574: 'A'-'Z' -575: 'a'-'z' -576: '0'-'9' -577: \u0100-\U0010ffff -578: . +573: \u00a0-\u00ff +574: \u0100-\U0010ffff +575: 'A'-'Z' +576: 'a'-'z' +577: '0'-'9' +578: \u0100-\U0010ffff +579: . */ diff --git a/internal/pkg/parsing/lexer/transitiontable.go b/internal/pkg/parsing/lexer/transitiontable.go index 08f2fd9302..8f0b772535 100644 --- a/internal/pkg/parsing/lexer/transitiontable.go +++ b/internal/pkg/parsing/lexer/transitiontable.go @@ -241,6 +241,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } @@ -1212,6 +1214,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } @@ -2980,6 +2984,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } @@ -4410,6 +4416,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } diff --git a/internal/pkg/parsing/mlr.bnf b/internal/pkg/parsing/mlr.bnf index baee22c5ca..f2b32fe2ce 100644 --- a/internal/pkg/parsing/mlr.bnf +++ b/internal/pkg/parsing/mlr.bnf @@ -54,7 +54,7 @@ // CHARACTER CLASSES // ---------------------------------------------------------------- -_letter : 'a'-'z' | 'A'-'Z' | '\u00a0'-'\u00ff' | '\u0100'-'\U0010FFFF'; +_letter : 'a'-'z' | 'A'-'Z' | '\u00a0'-'\u00ff' | '\u0100'-'\U0010ffff'; _decdig : '0'-'9' ; _hexdig : '0'-'9' | 'a'-'f' | 'A'-'F'; _octdig : '0'-'7' ; @@ -95,7 +95,8 @@ _string_literal_element | ( '\\' '*' ) | ( '\\' '0' ) | ( '\\' '1' ) | ( '\\' '2' ) | ( '\\' '3' ) | ( '\\' '4' ) | ( '\\' '5' ) | ( '\\' '6' ) | ( '\\' '7' ) | ( '\\' '8' ) | ( '\\' '9' ) - | '\u0100'-'\U0010FFFF' + | '\u00a0'-'\u00ff' + | '\u0100'-'\U0010ffff' ; string_literal : '"' {_string_literal_element} '"' ; diff --git a/todo.txt b/todo.txt index a84c57b2a1..67fb7ba158 100644 --- a/todo.txt +++ b/todo.txt @@ -7,6 +7,7 @@ RELEASES https://github.com/johnkerl/miller/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-documentation o fmt/unfmt/regex doc o FAQ/examples reorg + ? ??? for doc-string contents search -- ? & mlr help namegoeshere foo ... k strptime/882 k fmtifnum, & recursive fmtnum/fmtifnum