From 5f1355def913682dc32db6c68fe048e374656451 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 21 Feb 2022 13:43:44 -0500 Subject: [PATCH 1/2] Support Latin-1 supplement a0-ff as DSL string literals --- internal/pkg/parsing/mlr.bnf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/pkg/parsing/mlr.bnf b/internal/pkg/parsing/mlr.bnf index baee22c5ca..f2b32fe2ce 100644 --- a/internal/pkg/parsing/mlr.bnf +++ b/internal/pkg/parsing/mlr.bnf @@ -54,7 +54,7 @@ // CHARACTER CLASSES // ---------------------------------------------------------------- -_letter : 'a'-'z' | 'A'-'Z' | '\u00a0'-'\u00ff' | '\u0100'-'\U0010FFFF'; +_letter : 'a'-'z' | 'A'-'Z' | '\u00a0'-'\u00ff' | '\u0100'-'\U0010ffff'; _decdig : '0'-'9' ; _hexdig : '0'-'9' | 'a'-'f' | 'A'-'F'; _octdig : '0'-'7' ; @@ -95,7 +95,8 @@ _string_literal_element | ( '\\' '*' ) | ( '\\' '0' ) | ( '\\' '1' ) | ( '\\' '2' ) | ( '\\' '3' ) | ( '\\' '4' ) | ( '\\' '5' ) | ( '\\' '6' ) | ( '\\' '7' ) | ( '\\' '8' ) | ( '\\' '9' ) - | '\u0100'-'\U0010FFFF' + | '\u00a0'-'\u00ff' + | '\u0100'-'\U0010ffff' ; string_literal : '"' {_string_literal_element} '"' ; From bfb71c3e873995d40ab659bc4c76f5479a69be6d Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 21 Feb 2022 13:54:02 -0500 Subject: [PATCH 2/2] build artifacts for previous commit --- internal/pkg/parsing/lexer/lexer.go | 15 ++++++++------- internal/pkg/parsing/lexer/transitiontable.go | 8 ++++++++ todo.txt | 1 + 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/internal/pkg/parsing/lexer/lexer.go b/internal/pkg/parsing/lexer/lexer.go index 7894220c4e..1cfd61a36b 100644 --- a/internal/pkg/parsing/lexer/lexer.go +++ b/internal/pkg/parsing/lexer/lexer.go @@ -12,7 +12,7 @@ import ( const ( NoState = -1 NumStates = 328 - NumSymbols = 579 + NumSymbols = 580 ) type Lexer struct { @@ -702,10 +702,11 @@ Lexer symbols: 570: 'A'-'Z' 571: 'a'-'z' 572: '0'-'9' -573: \u0100-\U0010ffff -574: 'A'-'Z' -575: 'a'-'z' -576: '0'-'9' -577: \u0100-\U0010ffff -578: . +573: \u00a0-\u00ff +574: \u0100-\U0010ffff +575: 'A'-'Z' +576: 'a'-'z' +577: '0'-'9' +578: \u0100-\U0010ffff +579: . */ diff --git a/internal/pkg/parsing/lexer/transitiontable.go b/internal/pkg/parsing/lexer/transitiontable.go index 08f2fd9302..8f0b772535 100644 --- a/internal/pkg/parsing/lexer/transitiontable.go +++ b/internal/pkg/parsing/lexer/transitiontable.go @@ -241,6 +241,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } @@ -1212,6 +1214,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } @@ -2980,6 +2984,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } @@ -4410,6 +4416,8 @@ var TransTab = TransitionTable{ return 57 case r == 126: // ['~','~'] return 57 + case 160 <= r && r <= 255: // [\u00a0,\u00ff] + return 57 case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff] return 57 } diff --git a/todo.txt b/todo.txt index a84c57b2a1..67fb7ba158 100644 --- a/todo.txt +++ b/todo.txt @@ -7,6 +7,7 @@ RELEASES https://github.com/johnkerl/miller/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-documentation o fmt/unfmt/regex doc o FAQ/examples reorg + ? ??? for doc-string contents search -- ? & mlr help namegoeshere foo ... k strptime/882 k fmtifnum, & recursive fmtnum/fmtifnum