lexer: TOML: Support more integer and floating formats (#1832)

* Fix floating point format in TOML lexer in TOML, the decimal point must be surrounded by at least one digit on each side. * Support sign of number in TOML lexer * Support special float (nan/inf) in TOML lexer * Support hex, oct, bin integers in TOML lexer * Support underscores between digits in TOML lexer * Add more int/float examples to TOML visual sample * Use Num::Oct and non-captured group for oct int Co-authored-by: Tan Le <tan.le@hey.com> * Use Num::Bin and non-captured group for bin int Co-authored-by: Tan Le <tan.le@hey.com> * Use Num::Hex and non-captured group for hex int Co-authored-by: Tan Le <tan.le@hey.com> * Use non-captured group if capture is not needed Co-authored-by: Tan Le <tan.le@hey.com> * Use non-captured group if capture is not needed Co-authored-by: Tan Le <tan.le@hey.com> * Use non-captured group if capture is not needed Co-authored-by: Tan Le <tan.le@hey.com> Co-authored-by: Tan Le <tan.le@hey.com>
rouge-ruby · Jul 2, 2022 · 1facc23 · 1facc23
1 parent fdc083f
commit 1facc23
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 3 deletions.
diff --git a/lib/rouge/lexers/toml.rb b/lib/rouge/lexers/toml.rb
@@ -28,9 +28,14 @@ class TOML < RegexLexer
 
         rule %r/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/, Literal::Date
 
-        rule %r/(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?/, Num::Float
-        rule %r/\d+[eE][+-]?[0-9]+j?/, Num::Float
-        rule %r/\-?\d+/, Num::Integer
+        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float
+        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float
+        rule %r/[+-]?(?:nan|inf)/, Num::Float
+
+        rule %r/0x\h+(?:_\h+)*/, Num::Hex
+        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
+        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer
       end
 
       state :root do

diff --git a/spec/visual/samples/toml b/spec/visual/samples/toml
@@ -118,3 +118,23 @@ point = { x = 1, y = 2 }
 physical.color = "orange"
 physical.shape = "round"
 site."google.com" = true
+
+# floating point numbers
+float_space1 = 3.141_592_653
+float_space2 = 123_456.789_012e+1_000
+just_inf = inf
+pos_inf  = +inf
+neg_inf  = -inf
+just_nan = nan
+pos_nan  = +nan
+neg_nan  = -nan
+
+# integer formats
+dec         = 123456
+dec_spacing = 123_456
+hex         = 0xdeadBEEF
+hex_spacing = 0xdead_BEEF
+oct         = 0o755
+oct_spacing = 0o755_777
+bin         = 0b1100
+bin_spacing = 0b0101_1100