feat: Add utf8 support to Pattern Lexer to support utf8 chars (#13085)

grafana · May 31, 2024 · f6f8bab · f6f8bab
1 parent 21dd4af
commit f6f8bab
Show file tree

Hide file tree

Showing 7 changed files with 252 additions and 174 deletions.
diff --git a/pkg/logql/log/pattern/lexer.go b/pkg/logql/log/pattern/lexer.go
@@ -1,5 +1,7 @@
 package pattern
 
+import "unicode/utf8"
+
 type lexer struct {
 	data        []byte
 	p, pe, cs   int
@@ -57,6 +59,7 @@ func (lex *lexer) identifier(out *exprSymType) (int, error) {
 
 // nolint
 func (lex *lexer) literal(out *exprSymType) (int, error) {
-	out.literal = rune(lex.data[lex.ts])
+	decoded, _ := utf8.DecodeRune(lex.data[lex.ts:lex.te])
+	out.literal = decoded
 	return LITERAL, nil
 }
diff --git a/pkg/logql/log/pattern/lexer.rl b/pkg/logql/log/pattern/lexer.rl
@@ -13,11 +13,25 @@ package pattern
     }
 }%%
 
+%%{
+utf8 = (
+    0x00..0x7F |
+    0xC2..0xDF 0x80..0xBF |
+    0xE0 0xA0..0xBF 0x80..0xBF |
+    0xE1..0xEC 0x80..0xBF 0x80..0xBF |
+    0xED 0x80..0x9F 0x80..0xBF |
+    0xEE..0xEF 0x80..0xBF 0x80..0xBF |
+    0xF0 0x90..0xBF 0x80..0xBF 0x80..0xBF |
+    0xF1..0xF3 0x80..0xBF 0x80..0xBF 0x80..0xBF |
+    0xF4 0x80..0x8F 0x80..0xBF 0x80..0xBF
+);
+}%%
+
 const LEXER_ERROR = 0
 
 %%{
         identifier = '<' (alpha| '_') (alnum | '_' )* '>';
-        literal = any;
+        literal = utf8;
 }%%
 
 func (lex *lexer) Lex(out *exprSymType) int {