add WString support

Signed-off-by: Dirk Thomas <dirk-thomas@users.noreply.github.com>

rosidl_adapter/rosidl_adapter/msg/__init__.py

@@ -55,6 +55,7 @@ def convert_msg_to_idl(package_dir, package_name, input_file, output_dir):
     'float32': 'float',
     'float64': 'double',
     'string': 'string',
+    'wstring': 'wstring',
 }
 
 
@@ -69,6 +70,8 @@ def to_idl_literal(idl_type, value):
         return 'TRUE' if value else 'FALSE'
     if idl_type.startswith('string'):
         return string_to_idl_string_literal(value)
+    if idl_type.startswith('wstring'):
+        return string_to_idl_wstring_literal(value)
     return value
 
 
@@ -79,6 +82,10 @@ def string_to_idl_string_literal(string):
     return '"{0}"'.format(estr)
 
 
+def string_to_idl_wstring_literal(string):
+    return string_to_idl_string_literal(string)
+
+
 def get_include_file(base_type):
     if base_type.is_primitive_type():
         return None
@@ -90,7 +97,10 @@ def get_idl_type(type_):
         identifier = MSG_TYPE_TO_IDL[type_]
     elif type_.is_primitive_type():
         identifier = MSG_TYPE_TO_IDL[type_.type]
-        if identifier == 'string' and type_.string_upper_bound is not None:
+        if (
+            identifier in ('string', 'wstring') and
+            type_.string_upper_bound is not None
+        ):
             identifier += '<{type_.string_upper_bound}>'.format_map(locals())
     else:
         identifier = '{type_.pkg_name}::msg::{type_.type}' \

rosidl_adapter/rosidl_adapter/parser.py

@@ -51,7 +51,7 @@
     'int64',
     'uint64',
     'string',
-    # TODO reconsider wstring / u16string / u32string
+    'wstring',
     # TODO duration and time
     'duration',  # for compatibility only
     'time',  # for compatibility only
@@ -157,9 +157,12 @@ def __init__(self, type_string, context_package_name=None):
             self.type = type_string
             self.string_upper_bound = None
 
-        elif type_string.startswith('string%s' % STRING_UPPER_BOUND_TOKEN):
+        elif (
+            type_string.startswith('string%s' % STRING_UPPER_BOUND_TOKEN) or
+            type_string.startswith('wstring%s' % STRING_UPPER_BOUND_TOKEN)
+        ):
             self.pkg_name = None
-            self.type = 'string'
+            self.type = type_string.split(STRING_UPPER_BOUND_TOKEN, 1)[0]
             upper_bound_string = type_string[len(self.type) +
                                              len(STRING_UPPER_BOUND_TOKEN):]
 
@@ -321,7 +324,7 @@ def __eq__(self, other):
 
     def __str__(self):
         value = self.value
-        if self.type == 'string':
+        if self.type in ('string', 'wstring'):
             value = "'%s'" % value
         return '%s %s=%s' % (self.type, self.name, value)
 
@@ -356,7 +359,7 @@ def __str__(self):
         s = '%s %s' % (str(self.type), self.name)
         if self.default_value is not None:
             if self.type.is_primitive_type() and not self.type.is_array and \
-                    self.type.type == 'string':
+                    self.type.type in ('string', 'wstring'):
                 s += " '%s'" % self.default_value
             else:
                 s += ' %s' % self.default_value
@@ -571,7 +574,7 @@ def parse_value_string(type_, value_string):
                 "array value must start with '[' and end with ']'")
         elements_string = value_string[1:-1]
 
-        if type_.type == 'string':
+        if type_.type in ('string', 'wstring'):
             # String arrays need special processing as the comma can be part of a quoted string
             # and not a separator of array elements
             value_strings = parse_string_array_value_string(elements_string, type_.array_size)
@@ -725,7 +728,7 @@ def parse_primitive_value_string(type_, value_string):
 
         return value
 
-    if primitive_type == 'string':
+    if primitive_type in ('string', 'wstring'):
         # remove outer quotes to allow leading / trailing spaces in the string
         for quote in ['"', "'"]:
             if value_string.startswith(quote) and value_string.endswith(quote):

rosidl_generator_c/include/rosidl_generator_c/u16string_functions.h

@@ -45,6 +45,11 @@ bool
 rosidl_generator_c__U16String__assignn(
   rosidl_generator_c__U16String * str, const uint16_t * value, size_t n);
 
+ROSIDL_GENERATOR_C_PUBLIC
+bool
+rosidl_generator_c__U16String__assignn_from_char(
+  rosidl_generator_c__U16String * str, const char * value, size_t n);
+
 ROSIDL_GENERATOR_C_PUBLIC
 bool
 rosidl_generator_c__U16String__assign(
@@ -54,6 +59,11 @@ ROSIDL_GENERATOR_C_PUBLIC
 size_t
 rosidl_generator_c__U16String__len(const uint16_t * value);
 
+ROSIDL_GENERATOR_C_PUBLIC
+bool
+rosidl_generator_c__U16String__resize(
+  rosidl_generator_c__U16String * str, size_t n);
+
 ROSIDL_GENERATOR_C_PUBLIC
 bool
 rosidl_generator_c__U16String__Sequence__init(

rosidl_generator_c/rosidl_generator_c/__init__.py

@@ -126,6 +126,9 @@ def value_to_c(type_, value):
     if isinstance(type_, AbstractString):
         return '"%s"' % escape_string(value)
 
+    if isinstance(type_, AbstractWString):
+        return 'u"%s"' % escape_wstring(value)
+
     return basic_value_to_c(type_, value)
 
 
@@ -175,3 +178,7 @@ def escape_string(s):
     s = s.replace('\\', '\\\\')
     s = s.replace('"', r'\"')
     return s
+
+
+def escape_wstring(s):
+    return escape_string(s)

rosidl_generator_c/src/u16string_functions.c

@@ -94,6 +94,18 @@ rosidl_generator_c__U16String__assignn(
   return true;
 }
 
+bool
+rosidl_generator_c__U16String__assignn_from_char(
+  rosidl_generator_c__U16String * str, const char * value, size_t n)
+{
+  // since n represents the number of 8-bit characters it must be an even number
+  if (n % 2 != 0) {
+    return false;
+  }
+  return rosidl_generator_c__U16String__assignn(
+    str, (const uint16_t *)value, n / 2);
+}
+
 bool
 rosidl_generator_c__U16String__assign(
   rosidl_generator_c__U16String * str, const uint16_t * value)
@@ -115,6 +127,24 @@ rosidl_generator_c__U16String__len(const uint16_t * value)
   return len;
 }
 
+bool
+rosidl_generator_c__U16String__resize(
+  rosidl_generator_c__U16String * str, size_t n)
+{
+  if (!str) {
+    return false;
+  }
+  uint16_t * data = realloc(str->data, (n + 1) * sizeof(uint16_t));
+  if (!data) {
+    return false;
+  }
+  data[n] = 0;
+  str->data = data;
+  str->size = n;
+  str->capacity = n + 1;
+  return true;
+}
+
 bool
 rosidl_generator_c__U16String__Sequence__init(
   rosidl_generator_c__U16String__Sequence * sequence, size_t size)

rosidl_generator_cpp/resource/msg__struct.hpp.em

@@ -13,10 +13,12 @@
 @{
 from rosidl_generator_cpp import create_init_alloc_and_member_lists
 from rosidl_generator_cpp import escape_string
+from rosidl_generator_cpp import escape_wstring
 from rosidl_generator_cpp import msg_type_to_cpp
 from rosidl_generator_cpp import MSG_TYPE_TO_CPP
-from rosidl_parser.definition import AbstractGenericString
 from rosidl_parser.definition import AbstractNestedType
+from rosidl_parser.definition import AbstractString
+from rosidl_parser.definition import AbstractWString
 from rosidl_parser.definition import ACTION_FEEDBACK_SUFFIX
 from rosidl_parser.definition import ACTION_GOAL_SUFFIX
 from rosidl_parser.definition import ACTION_RESULT_SUFFIX
@@ -258,8 +260,10 @@ non_defaulted_zero_initialized_members = [
 
   // constant declarations
 @[for constant in message.constants]@
-@[ if isinstance(constant.type, AbstractGenericString)]@
+@[ if isinstance(constant.type, AbstractString)]@
   static const @(MSG_TYPE_TO_CPP['string']) @(constant.name);
+@[ elif isinstance(constant.type, AbstractWString)]@
+  static const @(MSG_TYPE_TO_CPP['wstring']) @(constant.name);
 @[ else]@
   static constexpr @(MSG_TYPE_TO_CPP[constant.type.typename]) @(constant.name) =
 @[  if isinstance(constant.type, BasicType) and constant.type.typename in (*INTEGER_TYPES, *CHARACTER_TYPES, BOOLEAN_TYPE, OCTET_TYPE)]@
@@ -335,10 +339,14 @@ using @(message.structure.namespaced_type.name) =
 
 // constant definitions
 @[for c in message.constants]@
-@[ if isinstance(c.type, AbstractGenericString)]@
+@[ if isinstance(c.type, AbstractString)]@
 template<typename ContainerAllocator>
 const @(MSG_TYPE_TO_CPP['string'])
 @(message.structure.namespaced_type.name)_<ContainerAllocator>::@(c.name) = "@(escape_string(c.value))";
+@[ elif isinstance(c.type, AbstractWString)]@
+template<typename ContainerAllocator>
+const @(MSG_TYPE_TO_CPP['wstring'])
+@(message.structure.namespaced_type.name)_<ContainerAllocator>::@(c.name) = u"@(escape_wstring(c.value))";
 @[ else ]@
 template<typename ContainerAllocator>
 constexpr @(MSG_TYPE_TO_CPP[c.type.typename]) @(message.structure.namespaced_type.name)_<ContainerAllocator>::@(c.name);

rosidl_generator_cpp/rosidl_generator_cpp/__init__.py

@@ -55,6 +55,8 @@ def generate_cpp(generator_arguments_file):
     'int64': 'int64_t',
     'string': 'std::basic_string<char, std::char_traits<char>, ' +
               'typename ContainerAllocator::template rebind<char>::other>',
+    'wstring': 'std::basic_string<char16_t, std::char_traits<char16_t>, ' +
+               'typename ContainerAllocator::template rebind<char16_t>::other>',
 }
 
 
@@ -75,7 +77,7 @@ def msg_type_only_to_cpp(type_):
     elif isinstance(type_, AbstractString):
         cpp_type = MSG_TYPE_TO_CPP['string']
     elif isinstance(type_, AbstractWString):
-        assert False, 'TBD'
+        cpp_type = MSG_TYPE_TO_CPP['wstring']
     elif isinstance(type_, NamespacedType):
         typename = '::'.join(type_.namespaced_name())
         cpp_type = typename + '_<ContainerAllocator>'
@@ -167,9 +169,12 @@ def primitive_value_to_cpp(type_, value):
         "Could not convert non-primitive type '%s' to CPP" % (type_)
     assert value is not None, "Value for type '%s' must not be None" % (type_)
 
-    if isinstance(type_, AbstractGenericString):
+    if isinstance(type_, AbstractString):
         return '"%s"' % escape_string(value)
 
+    if isinstance(type_, AbstractWString):
+        return 'u"%s"' % escape_wstring(value)
+
     if type_.typename == 'boolean':
         return 'true' if value else 'false'
 
@@ -221,6 +226,10 @@ def escape_string(s):
     return s
 
 
+def escape_wstring(s):
+    return escape_string(s)
+
+
 def create_init_alloc_and_member_lists(message):
     # A Member object represents the information we need to know to initialize
     # a single member of the class.

rosidl_parser/rosidl_parser/grammar.lark

@@ -66,9 +66,10 @@ _ESCAPE_SEQUENCES: "\\n" | "\\t" | "\\v" | "\\b" | "\\r" | "\\f" | "\\a" | "\\\\
 
 // 7.2.6.3 String Literals
 // string_literal: "\"" CHAR* "\""
-// replace precise rule based on the spec with regex for parsing performance
+// wide_string_literal:  "L\"" CHAR* "\""
+// replace precise rules based on the spec with regex for parsing performance
 string_literal: "\"\"" | "\"" /(\\\"|[^"])+/ "\""
-wide_string_literal:  "L\"" CHAR* "\""
+wide_string_literal: "L\"\"" | "L\"" /(\\\"|[^"])+/ "\""
 
 // 7.2.6.4 Floating-point Literals
 floating_pt_literal: DIGIT+ floating_pt_literal_dot DIGIT+ floating_pt_literal_e DIGIT*

rosidl_parser/rosidl_parser/parser.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import os
+import re
 import sys
 
 from lark import Lark
@@ -556,7 +558,11 @@ def get_const_expr_value(const_expr):
 
     if child.data == 'string_literal':
         assert not negate_value
-        return get_string_literal_value(child)
+        return get_string_literal_value(child, allow_unicode=False)
+
+    if child.data == 'wide_string_literal':
+        assert not negate_value
+        return get_string_literal_value(child, allow_unicode=True)
 
     assert False, 'Unsupported tree: ' + str(const_expr)
 
@@ -582,11 +588,14 @@ def get_floating_pt_literal_value(floating_pt_literal):
     return float(value)
 
 
-def get_string_literal_value(string_literal):
+def get_string_literal_value(string_literal, *, allow_unicode=False):
     assert len(string_literal.children) == 1
     child = string_literal.children[0]
     assert isinstance(child, Token)
     value = child.value
+
+    regex = _get_escape_sequences_regex(allow_unicode=allow_unicode)
+    value = regex.sub(_decode_escape_sequence, value)
     # unescape double quote and backslash if preceeded by a backslash
     i = 0
     while i < len(value):
@@ -595,3 +604,25 @@ def get_string_literal_value(string_literal):
                 value = value[:i] + value[i + 1:]
         i += 1
     return value
+
+
+def _get_escape_sequences_regex(*, allow_unicode):
+    # IDL Table 7-9: Escape sequences
+    pattern = '('
+    # newline, horizontal tab, vertical tab, backspace, carriage return,
+    # form feed, alert, backslash, question mark, single quote, double quote
+    pattern += '\\[ntvbrfa\\?\'"]'
+    # octal number
+    pattern += '|' + '\\[0-7]{1,3}'
+    # hexadecimal number
+    pattern += '|' + r'\\x.{1,2}'
+    if allow_unicode:
+        # unicode character
+        pattern += '|' + '\\u.{1,4}'
+    pattern += ')'
+
+    return re.compile(pattern)
+
+
+def _decode_escape_sequence(match):
+    return codecs.decode(match.group(0), 'unicode-escape')

rosidl_typesupport_introspection_cpp/resource/msg__type_support.cpp.em

@@ -65,7 +65,7 @@ if isinstance(member.type.value_type, BasicType):
 elif isinstance(member.type.value_type, AbstractString):
     type_ = 'std::string'
 elif isinstance(member.type.value_type, AbstractWString):
-    assert False, 'Unknown type: ' + str(member.type.value_type)
+    type_ = 'std::u16string'
 elif isinstance(member.type.value_type, NamespacedType):
     type_ = '::'.join(member.type.value_type.namespaced_name())
 }@