Improved string parsing

ClanCats · Mar 7, 2024 · 187e09e · 187e09e
1 parent 7acdb23
commit 187e09e
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 4 deletions.
diff --git a/src/ContainerParser/ContainerLexer.php b/src/ContainerParser/ContainerLexer.php
@@ -60,10 +60,6 @@ class ContainerLexer
      */
     protected array $tokenMap = 
     [
-        // strings
-        '/^"[^"\\\\]*(?:\\\\.[^"\\\\]*)*"/' => T::TOKEN_STRING,
-        "/^'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'/" => T::TOKEN_STRING,
-
         // numbers
         "/^[+\-]?([0-9]*[.])?[0-9]+/" => T::TOKEN_NUMBER,
 
@@ -156,6 +152,29 @@ protected function next()
             return false;
         }
 
+        // parse string seperatly
+        $char = $this->code[$this->offset];
+        if ($char === '"' || $char === "'") 
+        {
+            $string = $char;
+            $this->offset++;
+
+            while ($this->offset < $this->length) 
+            {
+                if ($this->code[$this->offset] === $char && $this->code[$this->offset - 1] !== "\\") {
+                    $string .= $char;
+                    break;
+                }
+
+                $string .= $this->code[$this->offset];
+                $this->offset++;
+            }
+
+            $this->offset++;
+
+            return new T($this->line + 1, T::TOKEN_STRING, $string, $this->filename);
+        }
+
         foreach ($this->tokenMap as $regex => $token) 
         {
             if (preg_match($regex, substr($this->code, $this->offset), $matches)) 

diff --git a/tests/ContainerParser/ContainerLexerTest.php b/tests/ContainerParser/ContainerLexerTest.php
@@ -38,6 +38,7 @@ public function testScalarString()
     {
         $this->assertTokenTypes("'hello'\"world\"", [T::TOKEN_STRING, T::TOKEN_STRING]);
 
+
         // escaping
         $string = $this->tokensFromCode('"James \' Bond"')[0];
         $this->assertEquals("James ' Bond", $string->getValue());
@@ -57,6 +58,15 @@ public function testScalarString()
         $this->assertTokenTypes("'\"\"'", [T::TOKEN_STRING]);
     }
 
+    public function testMultilineStrings()
+    {
+        $this->assertTokenTypes(<<<'CODE'
+        :foo: "myfunc() then
+            return 'bla bla'
+        end"
+        CODE, [T::TOKEN_PARAMETER, T::TOKEN_ASSIGN, T::TOKEN_SPACE, T::TOKEN_STRING]);
+    }
+
     public function testScalarNumber()
     {
         $this->assertTokenTypes("-1", [T::TOKEN_NUMBER]);

diff --git a/tests/ContainerParser/Parser/ParameterDefinitionParserTest.php b/tests/ContainerParser/Parser/ParameterDefinitionParserTest.php
@@ -144,4 +144,27 @@ public function testInvalidAssignValue()
         $this->expectException(\ClanCats\Container\Exceptions\ContainerParserException::class);
         $def = $this->parameterDefnitionNodeFromCode(':foo: @bar'); // actually i want this in the feature
     }
+
+    public function testMultilineStringWithEscapedQuates()
+    {
+        $content = file_get_contents(__DIR__ . '/../../ctn/multilineparamsq.ctn') ?: '';
+
+        $def = $this->parameterDefnitionNodeFromCode($content);
+        $this->assertEquals(<<<'EOS'
+        Know
+        Why
+        this
+        doesn't work
+
+        EOS, $def->getValue()->getRawValue());
+
+        $content = file_get_contents(__DIR__ . '/../../ctn/multilineparamdq.ctn') ?: '';
+
+        $def = $this->parameterDefnitionNodeFromCode($content);
+        $this->assertEquals(<<<'EOS'
+
+        Hey you should be able to say "Something"
+
+        EOS, $def->getValue()->getRawValue());
+    }
 }
diff --git a/tests/ctn/multilineparamdq.ctn b/tests/ctn/multilineparamdq.ctn
@@ -0,0 +1,3 @@
+:foo: "
+Hey you should be able to say \"Something\"
+"
diff --git a/tests/ctn/multilineparamsq.ctn b/tests/ctn/multilineparamsq.ctn
@@ -0,0 +1,6 @@
+
+:idont: 'Know
+Why
+this
+doesn\'t work
+'