allow to handle unicode strings with single/double quotes in parser #11

src/fiql_lexer.xrl

@@ -39,10 +39,10 @@ list_to_bool(Chars) ->
   to_bool(string:lowercase(list_to_binary(Chars))).
 
 from_double_quoted(Chars) ->
-  binary_to_list(re:replace(string:slice(list_to_binary(Chars), 1, string:length(list_to_binary(Chars))-2), "\\\\\"", "\"", [global, {return, binary}])).
+  unicode:characters_to_list(re:replace(string:slice(unicode:characters_to_binary(Chars), 1, string:length(unicode:characters_to_binary(Chars))-2), "\\\\\"", "\"", [global, {return, binary}])).
 
 from_single_quoted(Chars) ->
-  binary_to_list(re:replace(string:slice(list_to_binary(Chars), 1, string:length(list_to_binary(Chars))-2), "\\\\'", "'", [global, {return, binary}])).
+  unicode:characters_to_list(re:replace(string:slice(unicode:characters_to_binary(Chars), 1, string:length(unicode:characters_to_binary(Chars))-2), "\\\\'", "'", [global, {return, binary}])).
 
 strip_equals(Chars) ->
-  binary_to_list(string:slice(list_to_binary(Chars), 1, string:length(list_to_binary(Chars))-2)).
+  binary_to_list(string:slice(list_to_binary(Chars), 1, string:length(list_to_binary(Chars))-2)).

test/fiql_parser_test.exs

@@ -139,6 +139,22 @@ defmodule FIQLExParserTest do
              {:ok, {:op, {:selector_and_value, "my_selector", :equal, "àèìòù"}}}
   end
 
+  test "Selector with non ascii value and with single quotes" do
+    payload = "my_selector=='àèìòù'"
+    {:ok, tokens, _} = :fiql_lexer.string(to_charlist(payload))
+
+    assert :fiql_parser.parse(tokens) ==
+             {:ok, {:op, {:selector_and_value, "my_selector", :equal, "àèìòù"}}}
+  end
+
+  test "Selector with non ascii value and with double quotes" do
+    payload = "my_selector==\"àèìòù\""
+    {:ok, tokens, _} = :fiql_lexer.string(to_charlist(payload))
+
+    assert :fiql_parser.parse(tokens) ==
+             {:ok, {:op, {:selector_and_value, "my_selector", :equal, "àèìòù"}}}
+  end
+
   test "Selector with unicode value" do
     payload = "my_selector==€€€"
     {:ok, tokens, _} = :fiql_lexer.string(to_charlist(payload))
@@ -147,6 +163,14 @@ defmodule FIQLExParserTest do
              {:ok, {:op, {:selector_and_value, "my_selector", :equal, "€€€"}}}
   end
 
+  test "Selector with unicode value single quoted" do
+    payload = "my_selector=='ф'"
+    {:ok, tokens, _} = :fiql_lexer.string(to_charlist(payload))
+
+    assert :fiql_parser.parse(tokens) ==
+             {:ok, {:op, {:selector_and_value, "my_selector", :equal, "ф"}}}
+  end
+
   test "Selector with list value" do
     payload = "my_selector==(1, \"hello world\", (true, false))"
     {:ok, tokens, _} = :fiql_lexer.string(to_charlist(payload))