Type Description Codegen and Typesupport (rep2011) (#727)

* Generate code for and expose interface type descriptions

* Use the C definitions of type descriptions instead of generating cpp

* Add descriptions to srv support

* Add a few more description codegen smoke tests

* Add generator c package.xml dependencies

* Type description functions, not globals

* Encode w strings as utf 8

* Add ability to disable C description codegen

* Including referenced descriptions

* Type hash functions, new getter signatures

* Generating hashes in usable form and putting in description function

* Pass hash lookup to full_description

* Add test for copied sources against installed versions to detect out of date

* Generate full type description raw sources

Signed-off-by: Emerson Knapp <emerson.b.knapp@gmail.com>

rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake

@@ -32,7 +32,7 @@ function(rosidl_write_generator_arguments output_file)
   set(OPTIONAL_MULTI_VALUE_KEYWORDS
     "ROS_INTERFACE_DEPENDENCIES"  # since the dependencies can be empty
     "TARGET_DEPENDENCIES"
-    "TYPE_HASH_TUPLES"
+    "TYPE_DESCRIPTION_TUPLES"
     "INCLUDE_PATHS"
     "ADDITIONAL_FILES")
 

rosidl_generator_c/bin/rosidl_generator_c

@@ -29,9 +29,13 @@ def main(argv=sys.argv[1:]):
         '--generator-arguments-file',
         required=True,
         help='The location of the file containing the generator arguments')
+    parser.add_argument(
+        '--disable-description-codegen', action='store_true',
+        help='If set, disable the generation of static type description '
+             'code to reduce binary size.')
     args = parser.parse_args(argv)
 
-    generate_c(args.generator_arguments_file)
+    generate_c(args.generator_arguments_file, args.disable_description_codegen)
 
 
 if __name__ == '__main__':

rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake

@@ -33,6 +33,7 @@ foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES})
     "${_output_path}/${_parent_folder}/detail/${_header_name}__type_support.h"
   )
   list(APPEND _generated_sources
+    "${_output_path}/${_parent_folder}/detail/${_header_name}__description.c"
     "${_output_path}/${_parent_folder}/detail/${_header_name}__functions.c"
     "${_output_path}/${_parent_folder}/detail/${_header_name}__type_support.c"
   )
@@ -54,7 +55,10 @@ set(target_dependencies
   ${rosidl_generator_c_GENERATOR_FILES}
   "${rosidl_generator_c_TEMPLATE_DIR}/action__type_support.h.em"
   "${rosidl_generator_c_TEMPLATE_DIR}/action__type_support.c.em"
+  "${rosidl_generator_c_TEMPLATE_DIR}/empty__description.c.em"
+  "${rosidl_generator_c_TEMPLATE_DIR}/full__description.c.em"
   "${rosidl_generator_c_TEMPLATE_DIR}/idl.h.em"
+  "${rosidl_generator_c_TEMPLATE_DIR}/idl__description.c.em"
   "${rosidl_generator_c_TEMPLATE_DIR}/idl__functions.c.em"
   "${rosidl_generator_c_TEMPLATE_DIR}/idl__functions.h.em"
   "${rosidl_generator_c_TEMPLATE_DIR}/idl__struct.h.em"
@@ -74,6 +78,7 @@ foreach(dep ${target_dependencies})
   endif()
 endforeach()
 
+get_target_property(_target_sources ${rosidl_generate_interfaces_TARGET} SOURCES)
 set(generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_c__arguments.json")
 rosidl_write_generator_arguments(
   "${generator_arguments_file}"
@@ -83,16 +88,23 @@ rosidl_write_generator_arguments(
   OUTPUT_DIR "${_output_path}"
   TEMPLATE_DIR "${rosidl_generator_c_TEMPLATE_DIR}"
   TARGET_DEPENDENCIES ${target_dependencies}
-  TYPE_HASH_TUPLES "${${rosidl_generate_interfaces_TARGET}__HASH_TUPLES}"
+  TYPE_DESCRIPTION_TUPLES "${${rosidl_generate_interfaces_TARGET}__DESCRIPTION_TUPLES}"
+  ROS_INTERFACE_FILES "${_target_sources}"
 )
 
 find_package(Python3 REQUIRED COMPONENTS Interpreter)
 
+set(disable_description_codegen_arg)
+if(ROSIDL_GENERATOR_C_DISABLE_TYPE_DESCRIPTION_CODEGEN)
+  set(disable_description_codegen_arg "--disable-description-codegen")
+endif()
+
 add_custom_command(
   OUTPUT ${_generated_headers} ${_generated_sources}
   COMMAND Python3::Interpreter
   ARGS ${rosidl_generator_c_BIN}
   --generator-arguments-file "${generator_arguments_file}"
+  ${disable_description_codegen_arg}
   DEPENDS ${target_dependencies}
   COMMENT "Generating C code for ROS interfaces"
   VERBATIM

rosidl_generator_c/resource/empty__description.c.em

@@ -0,0 +1,54 @@
+@# Included from rosidl_generator_c/resource/idl__description.c.em
+@{
+from rosidl_generator_type_description import GET_DESCRIPTION_FUNC
+from rosidl_generator_type_description import GET_INDIVIDUAL_SOURCE_FUNC
+from rosidl_generator_type_description import GET_SOURCES_FUNC
+
+def typename_to_c(typename):
+  return typename.replace('/', '__')
+}@
+
+/// Define exported TypeDescriptions and TypeSources
+@[for msg, interface_type in [toplevel_type_description] + implicit_type_descriptions]@
+@{
+td_typename = msg['type_description']['type_name']
+td_c_typename = typename_to_c(td_typename)
+}@
+
+const rosidl_runtime_c__type_description__TypeDescription *
+@(td_c_typename)__@(GET_DESCRIPTION_FUNC)(
+  const rosidl_@(interface_type)_type_support_t * type_support)
+{
+  (void)type_support;
+  static const rosidl_runtime_c__type_description__TypeDescription description = {
+    {
+      {NULL, 0, 0},
+      {NULL, 0, 0},
+    },
+    {NULL, 0, 0},
+  };
+  return &description;
+}
+
+const rosidl_runtime_c__type_description__TypeSource *
+@(td_c_typename)__@(GET_INDIVIDUAL_SOURCE_FUNC)(
+  const rosidl_@(interface_type)_type_support_t * type_support)
+{
+  (void)type_support;
+  static const rosidl_runtime_c__type_description__TypeSource source = {
+    {NULL, 0, 0},
+    {NULL, 0, 0},
+    {NULL, 0, 0}
+  };
+  return &source;
+}
+
+const rosidl_runtime_c__type_description__TypeSource__Sequence *
+@(td_c_typename)__@(GET_SOURCES_FUNC)(
+  const rosidl_@(interface_type)_type_support_t * type_support)
+{
+  (void)type_support;
+  static const rosidl_runtime_c__type_description__TypeSource__Sequence sources = {NULL, 0, 0};
+  return &sources;
+}
+@[end for]@

rosidl_generator_c/resource/full__description.c.em

@@ -0,0 +1,237 @@
+@# Included from rosidl_generator_c/resource/idl__description.c.em
+@{
+from rosidl_generator_c import escape_string
+from rosidl_generator_c import idl_structure_type_to_c_include_prefix
+from rosidl_generator_c import type_hash_to_c_definition
+from rosidl_parser.definition import NamespacedType
+from rosidl_generator_type_description import FIELD_TYPE_ID_TO_NAME
+from rosidl_generator_type_description import GET_DESCRIPTION_FUNC
+from rosidl_generator_type_description import GET_HASH_FUNC
+from rosidl_generator_type_description import GET_INDIVIDUAL_SOURCE_FUNC
+from rosidl_generator_type_description import GET_SOURCES_FUNC
+
+def typename_to_c(typename):
+  return typename.replace('/', '__')
+
+def static_seq_n(varname, n):
+  """Statically define a runtime Sequence or String type."""
+  if n > 0:
+    return f'{{{varname}, {n}, {n}}}'
+  return '{NULL, 0, 0}'
+
+def static_seq(varname, values):
+  """Statically define a runtime Sequence or String type."""
+  if values:
+    return f'{{{varname}, {len(values)}, {len(values)}}}'
+  return '{NULL, 0, 0}'
+
+def utf8_encode(value_string):
+  from rosidl_generator_c import escape_string
+  # Slice removes the b'' from the representation.
+  return escape_string(repr(value_string.encode('utf-8'))[2:-1])
+
+implicit_type_names = set(td['type_description']['type_name'] for td, _ in implicit_type_descriptions)
+includes = set()
+toplevel_msg, _ = toplevel_type_description
+
+for referenced_td in toplevel_msg['referenced_type_descriptions']:
+    if referenced_td['type_name'] in implicit_type_names:
+        continue
+    names = referenced_td['type_name'].split('/')
+    _type = NamespacedType(names[:-1], names[-1])
+    include_prefix = idl_structure_type_to_c_include_prefix(_type, 'detail')
+    includes.add(include_prefix + '__functions.h')
+
+full_type_descriptions = [toplevel_type_description] + implicit_type_descriptions
+full_type_names = [t['type_description']['type_name'] for t, _ in full_type_descriptions]
+all_type_descriptions = [toplevel_msg['type_description']] + toplevel_msg['referenced_type_descriptions']
+
+toplevel_encoding = type_source_file.suffix[1:]
+with open(type_source_file, 'r', encoding='utf-8') as f:
+  raw_source_content = f.read()
+}@
+@
+#include <assert.h>
+#include <string.h>
+
+// Include directives for referenced types
+@[for header_file in includes]@
+#include "@(header_file)"
+@[end for]@
+
+@#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+@# Cache expected hashes for externally referenced types, for error checking
+// Hashes for external referenced types
+#ifndef NDEBUG
+@[for referenced_type_description in toplevel_msg['referenced_type_descriptions']]@
+@{
+type_name = referenced_type_description['type_name']
+c_typename = type_name.replace('/', '__')
+}@
+@[  if type_name not in full_type_names]@
+static const rosidl_type_hash_t @(c_typename)__EXPECTED_HASH = @(type_hash_to_c_definition(hash_lookup[type_name]));
+@[  end if]@
+@[end for]@
+#endif
+@#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+
+@#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+@# Names for all types
+@[for itype_description in all_type_descriptions]@
+static char @(typename_to_c(itype_description['type_name']))__TYPE_NAME[] = "@(itype_description['type_name'])";
+@[end for]@
+@#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+
+@#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+@# Define all values going into each local type
+@
+@[for msg, interface_type in full_type_descriptions]@
+@{
+itype_description = msg['type_description']
+td_typename = itype_description['type_name']
+td_c_typename = typename_to_c(td_typename)
+ref_tds = msg['referenced_type_descriptions']
+}@
+@
+// Define type names, field names, and default values
+@[  for field in itype_description['fields']]@
+static char @(td_c_typename)__FIELD_NAME__@(field['name'])[] = "@(field['name'])";
+@[    if field['default_value']]@
+static char @(td_c_typename)__DEFAULT_VALUE__@(field['name'])[] = "@(utf8_encode(field['default_value']))";
+@[    end if]@
+@[  end for]@
+
+@
+@[  if itype_description['fields']]@
+static rosidl_runtime_c__type_description__Field @(td_c_typename)__FIELDS[] = {
+@[    for field in itype_description['fields']]@
+  {
+    @(static_seq(f"{td_c_typename}__FIELD_NAME__{field['name']}", field['name'])),
+    {
+      rosidl_runtime_c__type_description__FieldType__@(FIELD_TYPE_ID_TO_NAME[field['type']['type_id']]),
+      @(field['type']['capacity']),
+      @(field['type']['string_capacity']),
+      @(static_seq(f"{typename_to_c(field['type']['nested_type_name'])}__TYPE_NAME", field['type']['nested_type_name'])),
+    },
+    @(static_seq(f"{td_c_typename}__DEFAULT_VALUE__{field['name']}", field['default_value'])),
+  },
+@[    end for]@
+};
+@[  end if]@
+@
+@[  if ref_tds]@
+
+static rosidl_runtime_c__type_description__IndividualTypeDescription @(td_c_typename)__REFERENCED_TYPE_DESCRIPTIONS[] = {
+@[    for ref_td in ref_tds]@
+  {
+    @(static_seq(f"{typename_to_c(ref_td['type_name'])}__TYPE_NAME", ref_td['type_name'])),
+    {NULL, 0, 0},
+  },
+@[    end for]@
+};
+@[  end if]@
+
+const rosidl_runtime_c__type_description__TypeDescription *
+@(td_c_typename)__@(GET_DESCRIPTION_FUNC)(
+  const rosidl_@(interface_type)_type_support_t * type_support)
+{
+  (void)type_support;
+  static bool constructed = false;
+  static const rosidl_runtime_c__type_description__TypeDescription description = {
+    {
+      @(static_seq(f'{td_c_typename}__TYPE_NAME', td_typename)),
+      @(static_seq(f'{td_c_typename}__FIELDS', msg['type_description']['fields'])),
+    },
+    @(static_seq(f'{td_c_typename}__REFERENCED_TYPE_DESCRIPTIONS', ref_tds)),
+  };
+  if (!constructed) {
+@[  for idx, ref_td in enumerate(ref_tds)]@
+@{
+c_typename = typename_to_c(ref_td['type_name'])
+}@
+@[    if ref_td['type_name'] not in full_type_names]@
+    assert(0 == memcmp(&@(c_typename)__EXPECTED_HASH, @(c_typename)__@(GET_HASH_FUNC)(NULL), sizeof(rosidl_type_hash_t)));
+@[    end if]@
+    description.referenced_type_descriptions.data[@(idx)].fields = @(c_typename)__@(GET_DESCRIPTION_FUNC)(NULL)->type_description.fields;
+@[  end for]@
+    constructed = true;
+  }
+  return &description;
+}
+@[end for]@
+@#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+
+@#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+@# Define individual raw sources
+@[if raw_source_content]@
+static char toplevel_type_raw_source[] =@
+@[  for line in raw_source_content.splitlines()[:-1]]
+  "@(utf8_encode(line))\n"@
+@[  end for]
+  "@(utf8_encode(raw_source_content.splitlines()[-1]))";
+@[end if]@
+
+static char @(toplevel_encoding)_encoding[] = "@(toplevel_encoding)";
+@[if implicit_type_descriptions]@
+static char implicit_encoding[] = "implicit";
+@[end if]@
+
+// Define all individual source functions
+@[for type_description_msg, interface_type in full_type_descriptions]@
+@{
+itype_description = type_description_msg['type_description']
+td_typename = itype_description['type_name']
+td_c_typename = typename_to_c(td_typename)
+if td_typename in implicit_type_names:
+  encoding = 'implicit'
+  contents_var = None
+  contents = None
+else:
+  encoding = toplevel_encoding
+  contents_var = 'toplevel_type_raw_source'
+  contents = raw_source_content
+}@
+
+const rosidl_runtime_c__type_description__TypeSource *
+@(td_c_typename)__@(GET_INDIVIDUAL_SOURCE_FUNC)(
+  const rosidl_@(interface_type)_type_support_t * type_support)
+{
+  (void)type_support;
+  static const rosidl_runtime_c__type_description__TypeSource source = {
+    @(static_seq(f'{td_c_typename}__TYPE_NAME', td_typename)),
+    @(static_seq(f'{encoding}_encoding', encoding)),
+    @(static_seq(contents_var, contents)),
+  };
+  return &source;
+}
+@[end for]@
+@#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+@
+@#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+@# Define full raw source sequences
+@[for type_description_msg, interface_type in full_type_descriptions]@
+@{
+ref_tds = type_description_msg['referenced_type_descriptions']
+num_sources = len(ref_tds) + 1
+td_c_typename = typename_to_c(type_description_msg['type_description']['type_name'])
+}@
+
+const rosidl_runtime_c__type_description__TypeSource__Sequence *
+@(td_c_typename)__@(GET_SOURCES_FUNC)(
+  const rosidl_@(interface_type)_type_support_t * type_support)
+{
+  (void)type_support;
+  static rosidl_runtime_c__type_description__TypeSource sources[@(num_sources)];
+  static const rosidl_runtime_c__type_description__TypeSource__Sequence source_sequence = @(static_seq_n('sources', num_sources));
+  static bool constructed = false;
+  if (!constructed) {
+    sources[0] = *@(td_c_typename)__@(GET_INDIVIDUAL_SOURCE_FUNC)(NULL),
+@[  for idx, ref_td in enumerate(ref_tds)]@
+    sources[@(idx + 1)] = *@(typename_to_c(ref_td['type_name']))__@(GET_INDIVIDUAL_SOURCE_FUNC)(NULL);
+@[  end for]@
+    constructed = true;
+  }
+  return &source_sequence;
+}
+@[end for]@
+@#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>