From a76cb62ad4adf1d7fef02fb10ff554dda7f17567 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Thu, 5 Apr 2018 17:28:05 -0700 Subject: [PATCH 01/38] DIRTY lexer with bugs rosservice: and rostopic: --- rcl/CMakeLists.txt | 1 + rcl/src/rcl/lexer.c | 616 ++++++++++++++++++++++++++++++++++++ rcl/src/rcl/lexer.h | 94 ++++++ rcl/test/CMakeLists.txt | 9 + rcl/test/rcl/test_lexer.cpp | 325 +++++++++++++++++++ 5 files changed, 1045 insertions(+) create mode 100644 rcl/src/rcl/lexer.c create mode 100644 rcl/src/rcl/lexer.h create mode 100644 rcl/test/rcl/test_lexer.cpp diff --git a/rcl/CMakeLists.txt b/rcl/CMakeLists.txt index 980e8bdc2..d1e9e982f 100644 --- a/rcl/CMakeLists.txt +++ b/rcl/CMakeLists.txt @@ -33,6 +33,7 @@ set(${PROJECT_NAME}_sources src/rcl/expand_topic_name.c src/rcl/graph.c src/rcl/guard_condition.c + src/rcl/lexer.c src/rcl/node.c src/rcl/publisher.c src/rcl/rcl.c diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c new file mode 100644 index 000000000..e1194926d --- /dev/null +++ b/rcl/src/rcl/lexer.c @@ -0,0 +1,616 @@ +// Copyright 2018 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#include "lexer.h" + + +/* This lexer is implemented as a finite state machine. + * + * Every character in a string causes a tranisition in the state machine. + * Transitions labeled '' are special. + * They cause the lexer to re analyze the same character in another state. + * + * dot graph of state machine + +digraph remapping_lexer { + rankdir=LR; + node [shape = box, fontsize = 7]; + T_TILDE_SLASH + T_URL_SERVICE + T_URL_TOPIC + T_COLON + T_NODE + T_NS + T_SEPARATOR + T_BR1 + T_BR2 + T_BR3 + T_BR4 + T_BR5 + T_BR6 + T_BR7 + T_BR8 + T_BR9 + T_TOKEN + T_FORWARD_SLASH + T_WILD_ONE + T_WILD_MULTI + T_EOF + T_NONE + node [shape = circle]; + S0 -> T_FORWARD_SLASH [ label = "/"]; + S0 -> S1 [ label = "\\"]; + S0 -> S2 [ label = "~"]; + S0 -> S3 [ label = "_" ]; + S0 -> S8 [ label = "a-qs-zA-Z"]; + S0 -> S10 [ label = "r"]; + S0 -> S29 [ label = "*"]; + S0 -> S30 [ label = ":"]; + S1 -> T_BR1 [ label = "1"]; + S1 -> T_BR2 [ label = "2"]; + S1 -> T_BR3 [ label = "3"]; + S1 -> T_BR4 [ label = "4"]; + S1 -> T_BR5 [ label = "5"]; + S1 -> T_BR6 [ label = "6"]; + S1 -> T_BR7 [ label = "7"]; + S1 -> T_BR8 [ label = "8"]; + S1 -> T_BR9 [ label = "9"]; + S2 -> T_TILDE_SLASH [ label ="/" ]; + S3 -> S4 [ label = "_" ]; + S3 -> S9 [ label = "", color = crimson, fontcolor = crimson]; + S4 -> S5 [ label = "n" ]; + S5 -> T_NS [ label = "s"]; + S5 -> S6 [ label = "o" ]; + S6 -> S7 [ label = "d" ]; + S7 -> T_NODE [ label = "e"]; + S8 -> T_TOKEN [ label = "", color=crimson, fontcolor=crimson]; + S8 -> S8 [ label = "a-zA-Z0-9"]; + S8 -> S9 [ label = "_"]; + S9 -> T_TOKEN [ label = "", color=crimson, fontcolor=crimson]; + S9 -> S8 [ label = "a-zA-Z0-9"]; + S10 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S10 -> S11 [ label = "o"]; + S11 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S11 -> S12 [ label = "s"]; + S12 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S12 -> S13 [ label = "t"]; + S12 -> S20 [ label = "s"]; + S13 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S13 -> S14 [ label = "o"]; + S14 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S14 -> S15 [ label = "p"]; + S15 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S15 -> S16 [ label = "i"]; + S16 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S16 -> S17 [ label = "c"]; + S17 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S17 -> S18 [ label = ":"]; + S18 -> S19 [ label = "/"]; + S19 -> T_URL_TOPIC [ label = "/"]; + S20 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S20 -> S21 [ label = "e"]; + S21 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S21 -> S22 [ label = "r"]; + S22 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S22 -> S23 [ label = "v"]; + S23 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S23 -> S24 [ label = "i"]; + S24 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S24 -> S25 [ label = "c"]; + S25 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S25 -> S26 [ label = "e"]; + S26 -> S27 [ label = ":"]; + S26 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S27 -> S28 [ label = "/"]; + S28 -> T_URL_SERVICE [ label = "/"]; + S29 -> T_WILD_MULTI[ label = "*"]; + S29 -> T_WILD_ONE [ label = "", color=crimson, fontcolor=crimson]; + S30 -> T_SEPARATOR [ label = "="]; + S30 -> T_COLON [ label = "", color=crimson, fontcolor=crimson]; +} +*/ + + +// Represents a transition from one state to another +typedef struct rcl_lexer_transition_t +{ + // Index of a state to transition to + const size_t to_state; + // Start of a range of chars (inclusive) which activates this transition + const char range_start; + // End of a range of chars (inclusive) which activates this transition + const char range_end; +} rcl_lexer_transition_t; + +// Represents a state that either has transitions or is terminal +typedef struct rcl_lexer_state_t +{ + // If no transition matches this causes a character to be analyzed a second time in another state + const size_t else_state; + // A value of a terminal if this is a terminal state + const rcl_lexer_terminal_t terminal; + // Transitions in the state machine (NULL value at end of array) + const rcl_lexer_transition_t transitions[11]; +} rcl_lexer_state_t; + +#define S0 0u +#define S1 1u +#define S2 2u +#define S3 3u +#define S4 4u +#define S5 5u +#define S6 6u +#define S7 7u +#define S8 8u +#define S9 9u +#define S10 10u +#define S11 11u +#define S12 12u +#define S13 13u +#define S14 14u +#define S15 15u +#define S16 16u +#define S17 17u +#define S18 18u +#define S19 19u +#define S20 20u +#define S21 21u +#define S22 22u +#define S23 23u +#define S24 24u +#define S25 25u +#define S26 26u +#define S27 27u +#define S28 28u +#define S29 29u +#define S30 30u + +#define T_TILDE_SLASH 31u +#define T_URL_SERVICE 32u +#define T_URL_TOPIC 33u +#define T_COLON 34u +#define T_NODE 35u +#define T_NS 36u +#define T_SEPARATOR 37u +#define T_BR1 38u +#define T_BR2 39u +#define T_BR3 40u +#define T_BR4 41u +#define T_BR5 42u +#define T_BR6 43u +#define T_BR7 44u +#define T_BR8 45u +#define T_BR9 46u +#define T_TOKEN 47u +#define T_FORWARD_SLASH 48u +#define T_WILD_ONE 49u +#define T_WILD_MULTI 50u +#define T_EOF 51u +#define T_NONE 52u + +// used to figure out if a state is terminal or not +#define FIRST_TERMINAL T_TILDE_SLASH + +#define END_TRANSITIONS {0, '\0', '\0'} + +static const rcl_lexer_state_t g_states[] = +{ + // Nonterminal states + // S0 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {T_FORWARD_SLASH, '/', '/' }, + {S1, '\\', '\\'}, + {S2, '~', '~'}, + {S3, '_', '_'}, + {S8, 'a', 'q'}, + {S8, 's', 'z'}, + {S8, 'A', 'Z'}, + {S10, 'r', 'r'}, + {S29, '*', '*'}, + {S30, ':', ':'}, + END_TRANSITIONS + } + }, + // S1 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {T_BR1, '1', '1'}, + {T_BR2, '2', '2'}, + {T_BR3, '3', '3'}, + {T_BR4, '4', '4'}, + {T_BR5, '5', '5'}, + {T_BR6, '6', '6'}, + {T_BR7, '7', '7'}, + {T_BR8, '8', '8'}, + {T_BR9, '9', '9'}, + END_TRANSITIONS + } + }, + // S2 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {T_TILDE_SLASH, '/', '/'}, + END_TRANSITIONS + } + }, + // S3 + { + S9, + RCL_TERMINAL_NONE, + { + {S4, '_', '_'}, + END_TRANSITIONS + } + }, + // S4 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {S5, 'n', 'n'}, + END_TRANSITIONS + } + }, + // S5 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {T_NS, 's', 's'}, + {S6, 'o', 'o'}, + END_TRANSITIONS + } + }, + // S6 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {S7, 'd', 'd'}, + END_TRANSITIONS + } + }, + // S7 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {T_NODE, 'e', 'e'}, + END_TRANSITIONS + } + }, + // S8 + { + T_TOKEN, + RCL_TERMINAL_NONE, + { + {S8, 'a', 'z'}, + {S8, 'A', 'Z'}, + {S8, '0', '9'}, + {S9, '_', '_'}, + END_TRANSITIONS + } + }, + // S9 + { + T_TOKEN, + RCL_TERMINAL_NONE, + { + {S8, 'a', 'z'}, + {S8, 'A', 'Z'}, + {S8, '0', '9'}, + END_TRANSITIONS + } + }, + // S10 + { + S8, + RCL_TERMINAL_NONE, + { + {S11, 'o', 'o'}, + END_TRANSITIONS + } + }, + // S11 + { + S8, + RCL_TERMINAL_NONE, + { + {S12, 's', 's'}, + END_TRANSITIONS + } + }, + // S12 + { + S8, + RCL_TERMINAL_NONE, + { + {S13, 't', 't'}, + {S20, 's', 's'}, + END_TRANSITIONS + } + }, + // S13 + { + S8, + RCL_TERMINAL_NONE, + { + {S14, 'o', 'o'}, + END_TRANSITIONS + } + }, + // S14 + { + S8, + RCL_TERMINAL_NONE, + { + {S15, 'p', 'p'}, + END_TRANSITIONS + } + }, + // S15 + { + S8, + RCL_TERMINAL_NONE, + { + {S16, 'i', 'i'}, + END_TRANSITIONS + } + }, + // S16 + { + S8, + RCL_TERMINAL_NONE, + { + {S17, 'c', 'c'}, + END_TRANSITIONS + } + }, + // S17 + { + S8, + RCL_TERMINAL_NONE, + { + {S18, ':', ':'}, + END_TRANSITIONS + } + }, + // S18 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {S19, '/', '/'}, + END_TRANSITIONS + } + }, + // S19 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {T_URL_TOPIC, '/', '/'}, + END_TRANSITIONS + } + }, + // S20 + { + S8, + RCL_TERMINAL_NONE, + { + {S21, 'e', 'e'}, + END_TRANSITIONS + } + }, + // S21 + { + S8, + RCL_TERMINAL_NONE, + { + {S22, 'r', 'r'}, + END_TRANSITIONS + } + }, + // S22 + { + S8, + RCL_TERMINAL_NONE, + { + {S23, 'v', 'v'}, + END_TRANSITIONS + } + }, + // S23 + { + S8, + RCL_TERMINAL_NONE, + { + {S24, 'i', 'i'}, + END_TRANSITIONS + } + }, + // S24 + { + S8, + RCL_TERMINAL_NONE, + { + {S25, 'c', 'c'}, + END_TRANSITIONS + } + }, + // S25 + { + S8, + RCL_TERMINAL_NONE, + { + {S26, 'e', 'e'}, + END_TRANSITIONS + } + }, + // S26 + { + S8, + RCL_TERMINAL_NONE, + { + {S27, ':', ':'}, + END_TRANSITIONS + } + }, + // S27 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {S28, '/', '/'}, + END_TRANSITIONS + } + }, + // S28 + { + T_NONE, + RCL_TERMINAL_NONE, + { + {T_URL_SERVICE, '/', '/'}, + END_TRANSITIONS + } + }, + // S29 + { + T_WILD_ONE, + RCL_TERMINAL_NONE, + { + {T_WILD_MULTI, '*', '*'}, + END_TRANSITIONS + } + }, + // S30 + { + T_COLON, + RCL_TERMINAL_NONE, + { + {T_SEPARATOR, '=', '='}, + END_TRANSITIONS + } + }, + // Terminal states + // 31 + {S0, RCL_TERMINAL_TILDE_SLASH, {END_TRANSITIONS}}, + // 32 + {S0, RCL_TERMINAL_URL_SERVICE, {END_TRANSITIONS}}, + // 33 + {S0, RCL_TERMINAL_URL_TOPIC, {END_TRANSITIONS}}, + // 34 + {S0, RCL_TERMINAL_COLON, {END_TRANSITIONS}}, + // 35 + {S0, RCL_TERMINAL_NODE, {END_TRANSITIONS}}, + // 36 + {S0, RCL_TERMINAL_NS, {END_TRANSITIONS}}, + // 37 + {S0, RCL_TERMINAL_SEPARATOR, {END_TRANSITIONS}}, + // 38 + {S0, RCL_TERMINAL_BR1, {END_TRANSITIONS}}, + // 39 + {S0, RCL_TERMINAL_BR2, {END_TRANSITIONS}}, + // 40 + {S0, RCL_TERMINAL_BR3, {END_TRANSITIONS}}, + // 41 + {S0, RCL_TERMINAL_BR4, {END_TRANSITIONS}}, + // 42 + {S0, RCL_TERMINAL_BR5, {END_TRANSITIONS}}, + // 43 + {S0, RCL_TERMINAL_BR6, {END_TRANSITIONS}}, + // 44 + {S0, RCL_TERMINAL_BR7, {END_TRANSITIONS}}, + // 45 + {S0, RCL_TERMINAL_BR8, {END_TRANSITIONS}}, + // 46 + {S0, RCL_TERMINAL_BR9, {END_TRANSITIONS}}, + // 47 + {S0, RCL_TERMINAL_TOKEN, {END_TRANSITIONS}}, + // 48 + {S0, RCL_TERMINAL_FORWARD_SLASH, {END_TRANSITIONS}}, + // 49 + {S0, RCL_TERMINAL_WILD_ONE, {END_TRANSITIONS}}, + // 50 + {S0, RCL_TERMINAL_WILD_MULTI, {END_TRANSITIONS}}, + // 51 + {S0, RCL_TERMINAL_EOF, {END_TRANSITIONS}}, + // 52 + {S0, RCL_TERMINAL_NONE, {END_TRANSITIONS}}, +}; + +rcl_ret_t +rcl_lexer_analyze( + const char * text, + rcl_lexer_terminal_t * terminal, + size_t * end_position) +{ + // TODO accept allocator just for error checking + // RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, allocator); + // RCL_CHECK_ARGUMENT_FOR_NULL(terminal, RCL_RET_INVALID_ARGUMENT, allocator); + // RCL_CHECK_ARGUMENT_FOR_NULL(end_position, RCL_RET_INVALID_ARGUMENT, allocator); + + *end_position = 0; + + if ('\0' == text[0]){ + // Early exit if string is empty + *terminal = RCL_TERMINAL_EOF; + return RCL_RET_OK; + } + + const rcl_lexer_state_t * state = &(g_states[S0]); + char current_char; + size_t next_state; + do { + current_char = text[*end_position]; + next_state = 0; + + // Loop through all transitions in current state and find one that matches + size_t transition_idx = 0; + const rcl_lexer_transition_t * transition; + do { + transition = &(state->transitions[transition_idx]); + if (transition->range_start <= current_char && transition->range_end >= current_char) { + next_state = transition->to_state; + break; + } + ++transition_idx; + } while (transition->to_state != 0); + + if (0 == next_state) { + // no transition found, try evaluating this char again in a different state + next_state = state->else_state; + if (T_NONE == next_state) { + // advance so a substring using end_position includes this char + ++(*end_position); + } + } else { + // Advance position in string to test next char + ++(*end_position); + } + + state = &(g_states[next_state]); + } while (next_state < FIRST_TERMINAL); + + *terminal = state->terminal; + return RCL_RET_OK; +} diff --git a/rcl/src/rcl/lexer.h b/rcl/src/rcl/lexer.h new file mode 100644 index 000000000..e03bbeb39 --- /dev/null +++ b/rcl/src/rcl/lexer.h @@ -0,0 +1,94 @@ +// Copyright 2018 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RCL__ARGUMENTS_H_ +#define RCL__ARGUMENTS_H_ + +#include + +#include "rcl/allocator.h" +#include "rcl/macros.h" +#include "rcl/types.h" +#include "rcl/visibility_control.h" + +#if __cplusplus +extern "C" +{ +#endif + +typedef enum rcl_lexer_terminal_t +{ + // Indicates no valid terminal was found + RCL_TERMINAL_NONE = 0, + // Indicates end of input has been reached + RCL_TERMINAL_EOF = 1, + // ~/ + RCL_TERMINAL_TILDE_SLASH = 2, + // rosservice:// + RCL_TERMINAL_URL_SERVICE = 3, + // rostopic:// + RCL_TERMINAL_URL_TOPIC = 4, + // : + RCL_TERMINAL_COLON = 5, + // __node + RCL_TERMINAL_NODE = 6, + // __ns + RCL_TERMINAL_NS = 7, + // := + RCL_TERMINAL_SEPARATOR = 8, + // \1 + RCL_TERMINAL_BR1 = 9, + // \2 + RCL_TERMINAL_BR2 = 10, + // \3 + RCL_TERMINAL_BR3 = 11, + // \4 + RCL_TERMINAL_BR4 = 12, + // \5 + RCL_TERMINAL_BR5 = 13, + // \6 + RCL_TERMINAL_BR6 = 14, + // \7 + RCL_TERMINAL_BR7 = 15, + // \8 + RCL_TERMINAL_BR8 = 16, + // \9 + RCL_TERMINAL_BR9 = 17, + // a name between slashes, must match (([a-zA-Z](_)?)|_)([0-9a-zA-Z](_)?)* + RCL_TERMINAL_TOKEN = 18, + // / + RCL_TERMINAL_FORWARD_SLASH = 19, + // * + RCL_TERMINAL_WILD_ONE = 20, + // ** + RCL_TERMINAL_WILD_MULTI = 21 +} rcl_lexer_terminal_t; + +// Analize string until one terminal is found +// If the string does not begin with a valid terminal, terminal will be RCL_TERMINAL_NONE +// +// Known Bugs +// 1. "rosservice:" returns T_NONE when it should T_TOKEN up to "rosservice" +// 2. "rostopic:" returns T_NONE when it should T_TOKEN up to "rostopic" +rcl_ret_t +rcl_lexer_analyze( + const char * text, + rcl_lexer_terminal_t * terminal, + size_t * end_position); + +#if __cplusplus +} +#endif + +#endif // RCL__LEXER_H_ diff --git a/rcl/test/CMakeLists.txt b/rcl/test/CMakeLists.txt index 20a5a78c6..a933320a3 100644 --- a/rcl/test/CMakeLists.txt +++ b/rcl/test/CMakeLists.txt @@ -67,6 +67,15 @@ function(test_target_function) AMENT_DEPENDENCIES ${rmw_implementation} ) + rcl_add_custom_gtest(test_lexer${target_suffix} + SRCS rcl/test_lexer.cpp + ENV ${extra_test_env} + APPEND_LIBRARY_DIRS ${extra_lib_dirs} + LIBRARIES ${PROJECT_NAME} ${extra_test_libraries} + INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/src/rcl + AMENT_DEPENDENCIES ${rmw_implementation} + ) + set(SKIP_TEST "") # TODO(wjwwood): remove this when the graph API works properly for connext dynamic if( diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp new file mode 100644 index 000000000..3d849b19e --- /dev/null +++ b/rcl/test/rcl/test_lexer.cpp @@ -0,0 +1,325 @@ +// Copyright 2018 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include "lexer.h" + +#ifdef RMW_IMPLEMENTATION +# define CLASSNAME_(NAME, SUFFIX) NAME ## __ ## SUFFIX +# define CLASSNAME(NAME, SUFFIX) CLASSNAME_(NAME, SUFFIX) +#else +# define CLASSNAME(NAME, SUFFIX) NAME +#endif + +class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test +{ +public: + void SetUp() + { + } + + void TearDown() + { + } +}; + +// Not using a function because with a macro gtest shows the line number where the macro is used +#define EXPECT_LEX(expected_terminal, expected_text, text) \ + do { \ + rcl_lexer_terminal_t actual_terminal; \ + size_t end_pos; \ + rcl_ret_t ret = rcl_lexer_analyze(text, &actual_terminal, &end_pos); \ + ASSERT_EQ(RCL_RET_OK, ret); \ + EXPECT_EQ(expected_terminal, actual_terminal); \ + std::string actual_text(text, end_pos); \ + EXPECT_STREQ(expected_text, actual_text.c_str()); \ + } while (false) + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_token) +{ + // Things get recognized as tokens whether input ends or non token characters come after them + EXPECT_LEX(RCL_TERMINAL_TOKEN, "foo", "foo"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "foo", "foo:"); + + // Check full range for starting character + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a", "a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "b", "b"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "c", "c"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "d", "d"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "e", "e"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "f", "f"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "g", "g"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "h", "h"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "i", "i"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "j", "j"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "k", "k"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "l", "l"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "m", "m"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "n", "n"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "o", "o"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "p", "p"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "q", "q"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "r", "r"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "s", "s"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "t", "t"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "u", "u"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "v", "v"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "w", "w"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "x", "x"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "y", "y"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "z", "z"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "A", "A"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "B", "B"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "C", "C"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "D", "D"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "E", "E"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "F", "F"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "G", "G"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "H", "H"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "I", "I"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "J", "J"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "K", "K"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "L", "L"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "M", "M"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "N", "N"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "O", "O"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "P", "P"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "Q", "Q"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "R", "R"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "S", "S"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "T", "T"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "U", "U"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "V", "V"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "W", "W"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "X", "X"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "Y", "Y"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "Z", "Z"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_", "_"); + + // Check banned characters adjacent to allowed ones in ASCII + EXPECT_LEX(RCL_TERMINAL_NONE, "@", "@"); + EXPECT_LEX(RCL_TERMINAL_NONE, "[", "["); + EXPECT_LEX(RCL_TERMINAL_NONE, "`", "`"); + EXPECT_LEX(RCL_TERMINAL_NONE, "{", "{"); + // Tokens cannot start with digits + EXPECT_LEX(RCL_TERMINAL_NONE, "0", "0"); + EXPECT_LEX(RCL_TERMINAL_NONE, "1", "1"); + EXPECT_LEX(RCL_TERMINAL_NONE, "2", "2"); + EXPECT_LEX(RCL_TERMINAL_NONE, "3", "3"); + EXPECT_LEX(RCL_TERMINAL_NONE, "4", "4"); + EXPECT_LEX(RCL_TERMINAL_NONE, "5", "5"); + EXPECT_LEX(RCL_TERMINAL_NONE, "6", "6"); + EXPECT_LEX(RCL_TERMINAL_NONE, "7", "7"); + EXPECT_LEX(RCL_TERMINAL_NONE, "8", "8"); + EXPECT_LEX(RCL_TERMINAL_NONE, "9", "9"); + + // Tokens may contain underscores + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_abcd", "_abcd"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "abcd_", "abcd_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "ab_cd", "ab_cd"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_a_b_c_d_", "_a_b_c_d_"); + + // Tokens cannot contain double underscores + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_a_", "_a__bcd"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a_", "a__bcd"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "A_", "A__bcd"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__a", "__a"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__A", "__A"); + + // Tokens may contain digits + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_0_", "_0_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_1_", "_1_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_2_", "_2_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_3_", "_3_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_4_", "_4_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_5_", "_5_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_6_", "_6_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_7_", "_7_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_8_", "_8_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_9_", "_9_"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a0a", "a0a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a1a", "a1a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a2a", "a2a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a3a", "a3a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a4a", "a4a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a5a", "a5a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a6a", "a6a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a7a", "a7a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a8a", "a8a"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a9a", "a9a"); + + // Tokens may end with digits + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_0", "_0"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_1", "_1"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_2", "_2"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_3", "_3"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_4", "_4"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_5", "_5"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_6", "_6"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_7", "_7"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_8", "_8"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_9", "_9"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a0", "a0"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a1", "a1"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a2", "a2"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a3", "a3"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a4", "a4"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a5", "a5"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a6", "a6"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a7", "a7"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a8", "a8"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "a9", "a9"); + + // Things that almost look like a url scheme but are actually tokens + EXPECT_LEX(RCL_TERMINAL_TOKEN, "ro", "ro"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "ros", "ros"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "ross", "ross"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosse", "rosse"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosser", "rosser"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosserv", "rosserv"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservi", "rosservi"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservic", "rosservic"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice"); + // Known Bugs + // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:"); + // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:/"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rost", "rost"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosto", "rosto"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostop", "rostop"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopi", "rostopi"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic"); + // Known bugs + // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:"); + // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:/"); + + // Tokens may contain uppercase characters + EXPECT_LEX(RCL_TERMINAL_TOKEN, "ABC", "ABC"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_DEF", "_DEF"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "_GHI_", "_GHI_"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_url_scheme) +{ + // No text after scheme + EXPECT_LEX(RCL_TERMINAL_URL_SERVICE, "rosservice://", "rosservice://"); + EXPECT_LEX(RCL_TERMINAL_URL_TOPIC, "rostopic://", "rostopic://"); + + // Some text after scheme + EXPECT_LEX(RCL_TERMINAL_URL_SERVICE, "rosservice://", "rosservice://abcd"); + EXPECT_LEX(RCL_TERMINAL_URL_SERVICE, "rosservice://", "rosservice:///"); + EXPECT_LEX(RCL_TERMINAL_URL_TOPIC, "rostopic://", "rostopic://abcd"); + EXPECT_LEX(RCL_TERMINAL_URL_TOPIC, "rostopic://", "rostopic:///"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_backreferences) +{ + // No text after backreference + EXPECT_LEX(RCL_TERMINAL_BR1, "\\1", "\\1"); + EXPECT_LEX(RCL_TERMINAL_BR2, "\\2", "\\2"); + EXPECT_LEX(RCL_TERMINAL_BR3, "\\3", "\\3"); + EXPECT_LEX(RCL_TERMINAL_BR4, "\\4", "\\4"); + EXPECT_LEX(RCL_TERMINAL_BR5, "\\5", "\\5"); + EXPECT_LEX(RCL_TERMINAL_BR6, "\\6", "\\6"); + EXPECT_LEX(RCL_TERMINAL_BR7, "\\7", "\\7"); + EXPECT_LEX(RCL_TERMINAL_BR8, "\\8", "\\8"); + EXPECT_LEX(RCL_TERMINAL_BR9, "\\9", "\\9"); + + // Some text after backreference + EXPECT_LEX(RCL_TERMINAL_BR1, "\\1", "\\1a"); + EXPECT_LEX(RCL_TERMINAL_BR2, "\\2", "\\2a"); + EXPECT_LEX(RCL_TERMINAL_BR3, "\\3", "\\3a"); + EXPECT_LEX(RCL_TERMINAL_BR4, "\\4", "\\4a"); + EXPECT_LEX(RCL_TERMINAL_BR5, "\\5", "\\5a"); + EXPECT_LEX(RCL_TERMINAL_BR6, "\\6", "\\6a"); + EXPECT_LEX(RCL_TERMINAL_BR7, "\\7", "\\7a"); + EXPECT_LEX(RCL_TERMINAL_BR8, "\\8", "\\8a"); + EXPECT_LEX(RCL_TERMINAL_BR9, "\\9", "\\9a"); + + // Not valid backreferences + EXPECT_LEX(RCL_TERMINAL_NONE, "\\0", "\\0"); + EXPECT_LEX(RCL_TERMINAL_NONE, "\\a", "\\a"); + EXPECT_LEX(RCL_TERMINAL_NONE, "\\Z", "\\Z"); + EXPECT_LEX(RCL_TERMINAL_NONE, "\\_", "\\_"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_forward_slash) +{ + EXPECT_LEX(RCL_TERMINAL_FORWARD_SLASH, "/", "/"); + EXPECT_LEX(RCL_TERMINAL_FORWARD_SLASH, "/", "//"); + EXPECT_LEX(RCL_TERMINAL_FORWARD_SLASH, "/", "/_"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_wildcards) +{ + EXPECT_LEX(RCL_TERMINAL_WILD_ONE, "*", "*"); + EXPECT_LEX(RCL_TERMINAL_WILD_ONE, "*", "*/"); + EXPECT_LEX(RCL_TERMINAL_WILD_MULTI, "**", "**"); + EXPECT_LEX(RCL_TERMINAL_WILD_MULTI, "**", "**/"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_colon) +{ + EXPECT_LEX(RCL_TERMINAL_COLON, ":", ":"); + EXPECT_LEX(RCL_TERMINAL_COLON, ":", ":r"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_separator) +{ + EXPECT_LEX(RCL_TERMINAL_SEPARATOR, ":=", ":="); + EXPECT_LEX(RCL_TERMINAL_SEPARATOR, ":=", ":=0"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_ns) +{ + // Has __ns + EXPECT_LEX(RCL_TERMINAL_NS, "__ns", "__ns"); + EXPECT_LEX(RCL_TERMINAL_NS, "__ns", "__nsssss"); + + // Things that are almost __ns + EXPECT_LEX(RCL_TERMINAL_NONE, "__", "__"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__n", "__n"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__n!", "__n!"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_node) +{ + // Has __node + EXPECT_LEX(RCL_TERMINAL_NODE, "__node", "__node"); + EXPECT_LEX(RCL_TERMINAL_NODE, "__node", "__nodessss"); + + // Things that are almost __node + EXPECT_LEX(RCL_TERMINAL_NONE, "__", "__"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__n", "__n"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__na", "__na"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__no", "__no"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__noa", "__noa"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__nod", "__nod"); + EXPECT_LEX(RCL_TERMINAL_NONE, "__noda", "__noda"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_tilde_slash) +{ + EXPECT_LEX(RCL_TERMINAL_TILDE_SLASH, "~/", "~/"); + EXPECT_LEX(RCL_TERMINAL_TILDE_SLASH, "~/", "~//"); + EXPECT_LEX(RCL_TERMINAL_NONE, "~", "~"); + EXPECT_LEX(RCL_TERMINAL_NONE, "~!", "~!"); +} + +TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_eof) +{ + EXPECT_LEX(RCL_TERMINAL_EOF, "", ""); +} From 7d01c3c6e1c8b0f118ff532707a8a2b71cd522ba Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 10:31:46 -0700 Subject: [PATCH 02/38] Fix bug with partial url schemes --- rcl/src/rcl/lexer.c | 176 +++++++++++++++++++++++------------- rcl/src/rcl/lexer.h | 7 +- rcl/test/rcl/test_lexer.cpp | 14 +-- 3 files changed, 125 insertions(+), 72 deletions(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index e1194926d..8cb554ac0 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -16,11 +16,23 @@ #include "lexer.h" -/* This lexer is implemented as a finite state machine. +/* The lexer tries to find a lexeme in a string. + * It looks at one character at a time, and uses that character's value to decide how to transition + * a state machine. + * A transition is taken if a character's ASCII value falls within its range. + * No transition's ranges overlap; there is never more than one matching transition. + * + * If no transition matches then the state machine takes an '' transition. + * Every state has exactly one '' transition. + * All states have an `` to T_NONE unless otherwise specified in the diagram below. + * + * When a transition is taken it causes the lexer to move to another character in the string. + * Normal transitions always move the lexer to the forwards one character. + * '' transitions may cause the lexer to move forwards 1, or backwards N. + * The movement M is written as M = 1 - N so it can be stored in an unsigned integer. + * For example, an `` transition with M = 0 moves the lexer forwards 1 character, M = 1 keeps + * the lexer at the current character, and M = 2 moves the lexer backwards one character. * - * Every character in a string causes a tranisition in the state machine. - * Transitions labeled '' are special. - * They cause the lexer to re analyze the same character in another state. * * dot graph of state machine @@ -69,56 +81,60 @@ digraph remapping_lexer { S1 -> T_BR9 [ label = "9"]; S2 -> T_TILDE_SLASH [ label ="/" ]; S3 -> S4 [ label = "_" ]; - S3 -> S9 [ label = "", color = crimson, fontcolor = crimson]; + S3 -> S9 [ label = "", color = crimson, fontcolor = crimson]; S4 -> S5 [ label = "n" ]; S5 -> T_NS [ label = "s"]; S5 -> S6 [ label = "o" ]; S6 -> S7 [ label = "d" ]; S7 -> T_NODE [ label = "e"]; - S8 -> T_TOKEN [ label = "", color=crimson, fontcolor=crimson]; + S8 -> T_TOKEN [ label = "", color=crimson, fontcolor=crimson]; S8 -> S8 [ label = "a-zA-Z0-9"]; S8 -> S9 [ label = "_"]; - S9 -> T_TOKEN [ label = "", color=crimson, fontcolor=crimson]; + S9 -> T_TOKEN [ label = "", color=crimson, fontcolor=crimson]; S9 -> S8 [ label = "a-zA-Z0-9"]; - S10 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S10 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S10 -> S11 [ label = "o"]; - S11 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S11 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S11 -> S12 [ label = "s"]; - S12 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S12 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S12 -> S13 [ label = "t"]; S12 -> S20 [ label = "s"]; - S13 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S13 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S13 -> S14 [ label = "o"]; - S14 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S14 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S14 -> S15 [ label = "p"]; - S15 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S15 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S15 -> S16 [ label = "i"]; - S16 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S16 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S16 -> S17 [ label = "c"]; - S17 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S17 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S17 -> S18 [ label = ":"]; S18 -> S19 [ label = "/"]; + S18 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S19 -> T_URL_TOPIC [ label = "/"]; - S20 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S19 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S20 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S20 -> S21 [ label = "e"]; - S21 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S21 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S21 -> S22 [ label = "r"]; - S22 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S22 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S22 -> S23 [ label = "v"]; - S23 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S23 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S23 -> S24 [ label = "i"]; - S24 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S24 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S24 -> S25 [ label = "c"]; - S25 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S25 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S25 -> S26 [ label = "e"]; S26 -> S27 [ label = ":"]; - S26 -> S8 [ label = "", color=crimson, fontcolor=crimson]; + S26 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S27 -> S28 [ label = "/"]; + S27 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S28 -> T_URL_SERVICE [ label = "/"]; + S28 -> S8 [ label = "", color=crimson, fontcolor=crimson]; S29 -> T_WILD_MULTI[ label = "*"]; - S29 -> T_WILD_ONE [ label = "", color=crimson, fontcolor=crimson]; + S29 -> T_WILD_ONE [ label = "", color=crimson, fontcolor=crimson]; S30 -> T_SEPARATOR [ label = "="]; - S30 -> T_COLON [ label = "", color=crimson, fontcolor=crimson]; + S30 -> T_COLON [ label = "", color=crimson, fontcolor=crimson]; } */ @@ -139,6 +155,8 @@ typedef struct rcl_lexer_state_t { // If no transition matches this causes a character to be analyzed a second time in another state const size_t else_state; + // Movement associated with taking else state + const size_t else_movement; // A value of a terminal if this is a terminal state const rcl_lexer_terminal_t terminal; // Transitions in the state machine (NULL value at end of array) @@ -211,6 +229,7 @@ static const rcl_lexer_state_t g_states[] = // S0 { T_NONE, + 0, RCL_TERMINAL_NONE, { {T_FORWARD_SLASH, '/', '/' }, @@ -229,6 +248,7 @@ static const rcl_lexer_state_t g_states[] = // S1 { T_NONE, + 0, RCL_TERMINAL_NONE, { {T_BR1, '1', '1'}, @@ -246,6 +266,7 @@ static const rcl_lexer_state_t g_states[] = // S2 { T_NONE, + 0, RCL_TERMINAL_NONE, { {T_TILDE_SLASH, '/', '/'}, @@ -255,6 +276,7 @@ static const rcl_lexer_state_t g_states[] = // S3 { S9, + 1, RCL_TERMINAL_NONE, { {S4, '_', '_'}, @@ -264,6 +286,7 @@ static const rcl_lexer_state_t g_states[] = // S4 { T_NONE, + 0, RCL_TERMINAL_NONE, { {S5, 'n', 'n'}, @@ -273,6 +296,7 @@ static const rcl_lexer_state_t g_states[] = // S5 { T_NONE, + 0, RCL_TERMINAL_NONE, { {T_NS, 's', 's'}, @@ -283,6 +307,7 @@ static const rcl_lexer_state_t g_states[] = // S6 { T_NONE, + 0, RCL_TERMINAL_NONE, { {S7, 'd', 'd'}, @@ -292,6 +317,7 @@ static const rcl_lexer_state_t g_states[] = // S7 { T_NONE, + 0, RCL_TERMINAL_NONE, { {T_NODE, 'e', 'e'}, @@ -301,6 +327,7 @@ static const rcl_lexer_state_t g_states[] = // S8 { T_TOKEN, + 1, RCL_TERMINAL_NONE, { {S8, 'a', 'z'}, @@ -313,6 +340,7 @@ static const rcl_lexer_state_t g_states[] = // S9 { T_TOKEN, + 1, RCL_TERMINAL_NONE, { {S8, 'a', 'z'}, @@ -324,6 +352,7 @@ static const rcl_lexer_state_t g_states[] = // S10 { S8, + 1, RCL_TERMINAL_NONE, { {S11, 'o', 'o'}, @@ -333,6 +362,7 @@ static const rcl_lexer_state_t g_states[] = // S11 { S8, + 1, RCL_TERMINAL_NONE, { {S12, 's', 's'}, @@ -342,6 +372,7 @@ static const rcl_lexer_state_t g_states[] = // S12 { S8, + 1, RCL_TERMINAL_NONE, { {S13, 't', 't'}, @@ -352,6 +383,7 @@ static const rcl_lexer_state_t g_states[] = // S13 { S8, + 1, RCL_TERMINAL_NONE, { {S14, 'o', 'o'}, @@ -361,6 +393,7 @@ static const rcl_lexer_state_t g_states[] = // S14 { S8, + 1, RCL_TERMINAL_NONE, { {S15, 'p', 'p'}, @@ -370,6 +403,7 @@ static const rcl_lexer_state_t g_states[] = // S15 { S8, + 1, RCL_TERMINAL_NONE, { {S16, 'i', 'i'}, @@ -379,6 +413,7 @@ static const rcl_lexer_state_t g_states[] = // S16 { S8, + 1, RCL_TERMINAL_NONE, { {S17, 'c', 'c'}, @@ -388,6 +423,7 @@ static const rcl_lexer_state_t g_states[] = // S17 { S8, + 1, RCL_TERMINAL_NONE, { {S18, ':', ':'}, @@ -396,7 +432,8 @@ static const rcl_lexer_state_t g_states[] = }, // S18 { - T_NONE, + S8, + 2, RCL_TERMINAL_NONE, { {S19, '/', '/'}, @@ -405,7 +442,8 @@ static const rcl_lexer_state_t g_states[] = }, // S19 { - T_NONE, + S8, + 3, RCL_TERMINAL_NONE, { {T_URL_TOPIC, '/', '/'}, @@ -415,6 +453,7 @@ static const rcl_lexer_state_t g_states[] = // S20 { S8, + 1, RCL_TERMINAL_NONE, { {S21, 'e', 'e'}, @@ -424,6 +463,7 @@ static const rcl_lexer_state_t g_states[] = // S21 { S8, + 1, RCL_TERMINAL_NONE, { {S22, 'r', 'r'}, @@ -433,6 +473,7 @@ static const rcl_lexer_state_t g_states[] = // S22 { S8, + 1, RCL_TERMINAL_NONE, { {S23, 'v', 'v'}, @@ -442,6 +483,7 @@ static const rcl_lexer_state_t g_states[] = // S23 { S8, + 1, RCL_TERMINAL_NONE, { {S24, 'i', 'i'}, @@ -451,6 +493,7 @@ static const rcl_lexer_state_t g_states[] = // S24 { S8, + 1, RCL_TERMINAL_NONE, { {S25, 'c', 'c'}, @@ -460,6 +503,7 @@ static const rcl_lexer_state_t g_states[] = // S25 { S8, + 1, RCL_TERMINAL_NONE, { {S26, 'e', 'e'}, @@ -469,6 +513,7 @@ static const rcl_lexer_state_t g_states[] = // S26 { S8, + 1, RCL_TERMINAL_NONE, { {S27, ':', ':'}, @@ -477,7 +522,8 @@ static const rcl_lexer_state_t g_states[] = }, // S27 { - T_NONE, + S8, + 2, RCL_TERMINAL_NONE, { {S28, '/', '/'}, @@ -486,7 +532,8 @@ static const rcl_lexer_state_t g_states[] = }, // S28 { - T_NONE, + S8, + 3, RCL_TERMINAL_NONE, { {T_URL_SERVICE, '/', '/'}, @@ -496,6 +543,7 @@ static const rcl_lexer_state_t g_states[] = // S29 { T_WILD_ONE, + 1, RCL_TERMINAL_NONE, { {T_WILD_MULTI, '*', '*'}, @@ -505,6 +553,7 @@ static const rcl_lexer_state_t g_states[] = // S30 { T_COLON, + 1, RCL_TERMINAL_NONE, { {T_SEPARATOR, '=', '='}, @@ -513,63 +562,63 @@ static const rcl_lexer_state_t g_states[] = }, // Terminal states // 31 - {S0, RCL_TERMINAL_TILDE_SLASH, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_TILDE_SLASH, {END_TRANSITIONS}}, // 32 - {S0, RCL_TERMINAL_URL_SERVICE, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_URL_SERVICE, {END_TRANSITIONS}}, // 33 - {S0, RCL_TERMINAL_URL_TOPIC, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_URL_TOPIC, {END_TRANSITIONS}}, // 34 - {S0, RCL_TERMINAL_COLON, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_COLON, {END_TRANSITIONS}}, // 35 - {S0, RCL_TERMINAL_NODE, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_NODE, {END_TRANSITIONS}}, // 36 - {S0, RCL_TERMINAL_NS, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_NS, {END_TRANSITIONS}}, // 37 - {S0, RCL_TERMINAL_SEPARATOR, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_SEPARATOR, {END_TRANSITIONS}}, // 38 - {S0, RCL_TERMINAL_BR1, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR1, {END_TRANSITIONS}}, // 39 - {S0, RCL_TERMINAL_BR2, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR2, {END_TRANSITIONS}}, // 40 - {S0, RCL_TERMINAL_BR3, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR3, {END_TRANSITIONS}}, // 41 - {S0, RCL_TERMINAL_BR4, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR4, {END_TRANSITIONS}}, // 42 - {S0, RCL_TERMINAL_BR5, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR5, {END_TRANSITIONS}}, // 43 - {S0, RCL_TERMINAL_BR6, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR6, {END_TRANSITIONS}}, // 44 - {S0, RCL_TERMINAL_BR7, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR7, {END_TRANSITIONS}}, // 45 - {S0, RCL_TERMINAL_BR8, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR8, {END_TRANSITIONS}}, // 46 - {S0, RCL_TERMINAL_BR9, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_BR9, {END_TRANSITIONS}}, // 47 - {S0, RCL_TERMINAL_TOKEN, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_TOKEN, {END_TRANSITIONS}}, // 48 - {S0, RCL_TERMINAL_FORWARD_SLASH, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_FORWARD_SLASH, {END_TRANSITIONS}}, // 49 - {S0, RCL_TERMINAL_WILD_ONE, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_WILD_ONE, {END_TRANSITIONS}}, // 50 - {S0, RCL_TERMINAL_WILD_MULTI, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_WILD_MULTI, {END_TRANSITIONS}}, // 51 - {S0, RCL_TERMINAL_EOF, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_EOF, {END_TRANSITIONS}}, // 52 - {S0, RCL_TERMINAL_NONE, {END_TRANSITIONS}}, + {S0, 0, RCL_TERMINAL_NONE, {END_TRANSITIONS}}, }; rcl_ret_t rcl_lexer_analyze( const char * text, rcl_lexer_terminal_t * terminal, - size_t * end_position) + size_t * length) { // TODO accept allocator just for error checking // RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, allocator); // RCL_CHECK_ARGUMENT_FOR_NULL(terminal, RCL_RET_INVALID_ARGUMENT, allocator); - // RCL_CHECK_ARGUMENT_FOR_NULL(end_position, RCL_RET_INVALID_ARGUMENT, allocator); + // RCL_CHECK_ARGUMENT_FOR_NULL(length, RCL_RET_INVALID_ARGUMENT, allocator); - *end_position = 0; + *length = 0; if ('\0' == text[0]){ // Early exit if string is empty @@ -580,9 +629,11 @@ rcl_lexer_analyze( const rcl_lexer_state_t * state = &(g_states[S0]); char current_char; size_t next_state; + size_t movement; do { - current_char = text[*end_position]; + current_char = text[*length]; next_state = 0; + movement = 0; // Loop through all transitions in current state and find one that matches size_t transition_idx = 0; @@ -597,15 +648,18 @@ rcl_lexer_analyze( } while (transition->to_state != 0); if (0 == next_state) { - // no transition found, try evaluating this char again in a different state + // no transition found, take the else transition next_state = state->else_state; - if (T_NONE == next_state) { - // advance so a substring using end_position includes this char - ++(*end_position); - } - } else { + movement = state->else_movement; + } + + if (0 == movement) { // Advance position in string to test next char - ++(*end_position); + ++(*length); + } else { + // Go backwards in string + *length -= movement - 1; + // Error if movement would cause length to overflow } state = &(g_states[next_state]); diff --git a/rcl/src/rcl/lexer.h b/rcl/src/rcl/lexer.h index e03bbeb39..16478e2a2 100644 --- a/rcl/src/rcl/lexer.h +++ b/rcl/src/rcl/lexer.h @@ -77,15 +77,12 @@ typedef enum rcl_lexer_terminal_t // Analize string until one terminal is found // If the string does not begin with a valid terminal, terminal will be RCL_TERMINAL_NONE -// -// Known Bugs -// 1. "rosservice:" returns T_NONE when it should T_TOKEN up to "rosservice" -// 2. "rostopic:" returns T_NONE when it should T_TOKEN up to "rostopic" +// If the first character is '\0', the terminal will be RCL_TERMINAL_EOF rcl_ret_t rcl_lexer_analyze( const char * text, rcl_lexer_terminal_t * terminal, - size_t * end_position); + size_t * length); #if __cplusplus } diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index 3d849b19e..3ea7e531b 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -194,17 +194,19 @@ TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_token) EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservi", "rosservi"); EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservic", "rosservic"); EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice"); - // Known Bugs - // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:"); - // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:/"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:="); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:/"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:/a"); EXPECT_LEX(RCL_TERMINAL_TOKEN, "rost", "rost"); EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosto", "rosto"); EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostop", "rostop"); EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopi", "rostopi"); EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic"); - // Known bugs - // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:"); - // EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:/"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:="); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:/"); + EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:/a"); // Tokens may contain uppercase characters EXPECT_LEX(RCL_TERMINAL_TOKEN, "ABC", "ABC"); From 974b44f2fb8fd97f142b2f24bc62ad35c18e76a4 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 10:42:12 -0700 Subject: [PATCH 03/38] Style fixes --- rcl/src/rcl/lexer.c | 12 +++++------- rcl/src/rcl/lexer.h | 4 ++-- rcl/test/CMakeLists.txt | 2 +- rcl/test/rcl/test_lexer.cpp | 17 +++++++++-------- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index 8cb554ac0..bea4d6684 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. - -#include "lexer.h" - +#include "./lexer.h" /* The lexer tries to find a lexeme in a string. * It looks at one character at a time, and uses that character's value to decide how to transition @@ -232,7 +230,7 @@ static const rcl_lexer_state_t g_states[] = 0, RCL_TERMINAL_NONE, { - {T_FORWARD_SLASH, '/', '/' }, + {T_FORWARD_SLASH, '/', '/'}, {S1, '\\', '\\'}, {S2, '~', '~'}, {S3, '_', '_'}, @@ -613,14 +611,14 @@ rcl_lexer_analyze( rcl_lexer_terminal_t * terminal, size_t * length) { - // TODO accept allocator just for error checking + // TODO(sloretz) accept allocator just for error checking // RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, allocator); // RCL_CHECK_ARGUMENT_FOR_NULL(terminal, RCL_RET_INVALID_ARGUMENT, allocator); // RCL_CHECK_ARGUMENT_FOR_NULL(length, RCL_RET_INVALID_ARGUMENT, allocator); *length = 0; - if ('\0' == text[0]){ + if ('\0' == text[0]) { // Early exit if string is empty *terminal = RCL_TERMINAL_EOF; return RCL_RET_OK; @@ -659,7 +657,7 @@ rcl_lexer_analyze( } else { // Go backwards in string *length -= movement - 1; - // Error if movement would cause length to overflow + // TODO(sloretz) Error if movement would cause length to overflow } state = &(g_states[next_state]); diff --git a/rcl/src/rcl/lexer.h b/rcl/src/rcl/lexer.h index 16478e2a2..855f21d71 100644 --- a/rcl/src/rcl/lexer.h +++ b/rcl/src/rcl/lexer.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef RCL__ARGUMENTS_H_ -#define RCL__ARGUMENTS_H_ +#ifndef RCL__LEXER_H_ +#define RCL__LEXER_H_ #include diff --git a/rcl/test/CMakeLists.txt b/rcl/test/CMakeLists.txt index a933320a3..90c6deb16 100644 --- a/rcl/test/CMakeLists.txt +++ b/rcl/test/CMakeLists.txt @@ -72,7 +72,7 @@ function(test_target_function) ENV ${extra_test_env} APPEND_LIBRARY_DIRS ${extra_lib_dirs} LIBRARIES ${PROJECT_NAME} ${extra_test_libraries} - INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/src/rcl + INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/src AMENT_DEPENDENCIES ${rmw_implementation} ) diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index 3ea7e531b..a5d5de61a 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -16,7 +16,8 @@ #include -#include "lexer.h" +// test include directory has src +#include "rcl/lexer.h" #ifdef RMW_IMPLEMENTATION # define CLASSNAME_(NAME, SUFFIX) NAME ## __ ## SUFFIX @@ -40,13 +41,13 @@ class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test // Not using a function because with a macro gtest shows the line number where the macro is used #define EXPECT_LEX(expected_terminal, expected_text, text) \ do { \ - rcl_lexer_terminal_t actual_terminal; \ - size_t end_pos; \ - rcl_ret_t ret = rcl_lexer_analyze(text, &actual_terminal, &end_pos); \ - ASSERT_EQ(RCL_RET_OK, ret); \ - EXPECT_EQ(expected_terminal, actual_terminal); \ - std::string actual_text(text, end_pos); \ - EXPECT_STREQ(expected_text, actual_text.c_str()); \ + rcl_lexer_terminal_t actual_terminal; \ + size_t end_pos; \ + rcl_ret_t ret = rcl_lexer_analyze(text, &actual_terminal, &end_pos); \ + ASSERT_EQ(RCL_RET_OK, ret); \ + EXPECT_EQ(expected_terminal, actual_terminal); \ + std::string actual_text(text, end_pos); \ + EXPECT_STREQ(expected_text, actual_text.c_str()); \ } while (false) TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_token) From cb5210bfaadaed46156afb3e67c228aaec0112d0 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 10:44:59 -0700 Subject: [PATCH 04/38] Moved lexer.h to satisfy cpplint --- rcl/{src => include}/rcl/lexer.h | 0 rcl/src/rcl/lexer.c | 2 +- rcl/test/CMakeLists.txt | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) rename rcl/{src => include}/rcl/lexer.h (100%) diff --git a/rcl/src/rcl/lexer.h b/rcl/include/rcl/lexer.h similarity index 100% rename from rcl/src/rcl/lexer.h rename to rcl/include/rcl/lexer.h diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index bea4d6684..e537078e2 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "./lexer.h" +#include "rcl/lexer.h" /* The lexer tries to find a lexeme in a string. * It looks at one character at a time, and uses that character's value to decide how to transition diff --git a/rcl/test/CMakeLists.txt b/rcl/test/CMakeLists.txt index 90c6deb16..4ae5ed8ea 100644 --- a/rcl/test/CMakeLists.txt +++ b/rcl/test/CMakeLists.txt @@ -72,7 +72,6 @@ function(test_target_function) ENV ${extra_test_env} APPEND_LIBRARY_DIRS ${extra_lib_dirs} LIBRARIES ${PROJECT_NAME} ${extra_test_libraries} - INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/src AMENT_DEPENDENCIES ${rmw_implementation} ) From 2324f88b9b941026bca96d5531285e3ec4a6875e Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 11:03:39 -0700 Subject: [PATCH 05/38] moved terminals to their own array to reduce code size --- rcl/src/rcl/lexer.c | 151 ++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 88 deletions(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index e537078e2..ee32bb6e6 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -155,8 +155,6 @@ typedef struct rcl_lexer_state_t const size_t else_state; // Movement associated with taking else state const size_t else_movement; - // A value of a terminal if this is a terminal state - const rcl_lexer_terminal_t terminal; // Transitions in the state machine (NULL value at end of array) const rcl_lexer_transition_t transitions[11]; } rcl_lexer_state_t; @@ -192,6 +190,7 @@ typedef struct rcl_lexer_state_t #define S28 28u #define S29 29u #define S30 30u +#define LAST_STATE S30 #define T_TILDE_SLASH 31u #define T_URL_SERVICE 32u @@ -218,17 +217,18 @@ typedef struct rcl_lexer_state_t // used to figure out if a state is terminal or not #define FIRST_TERMINAL T_TILDE_SLASH +#define LAST_TERMINAL T_NONE +// Used to mark where the last transition is in a state #define END_TRANSITIONS {0, '\0', '\0'} -static const rcl_lexer_state_t g_states[] = +static const rcl_lexer_state_t g_states[LAST_STATE + 1] = { // Nonterminal states // S0 { T_NONE, 0, - RCL_TERMINAL_NONE, { {T_FORWARD_SLASH, '/', '/'}, {S1, '\\', '\\'}, @@ -247,7 +247,6 @@ static const rcl_lexer_state_t g_states[] = { T_NONE, 0, - RCL_TERMINAL_NONE, { {T_BR1, '1', '1'}, {T_BR2, '2', '2'}, @@ -265,7 +264,6 @@ static const rcl_lexer_state_t g_states[] = { T_NONE, 0, - RCL_TERMINAL_NONE, { {T_TILDE_SLASH, '/', '/'}, END_TRANSITIONS @@ -275,7 +273,6 @@ static const rcl_lexer_state_t g_states[] = { S9, 1, - RCL_TERMINAL_NONE, { {S4, '_', '_'}, END_TRANSITIONS @@ -285,7 +282,6 @@ static const rcl_lexer_state_t g_states[] = { T_NONE, 0, - RCL_TERMINAL_NONE, { {S5, 'n', 'n'}, END_TRANSITIONS @@ -295,7 +291,6 @@ static const rcl_lexer_state_t g_states[] = { T_NONE, 0, - RCL_TERMINAL_NONE, { {T_NS, 's', 's'}, {S6, 'o', 'o'}, @@ -306,7 +301,6 @@ static const rcl_lexer_state_t g_states[] = { T_NONE, 0, - RCL_TERMINAL_NONE, { {S7, 'd', 'd'}, END_TRANSITIONS @@ -316,7 +310,6 @@ static const rcl_lexer_state_t g_states[] = { T_NONE, 0, - RCL_TERMINAL_NONE, { {T_NODE, 'e', 'e'}, END_TRANSITIONS @@ -326,7 +319,6 @@ static const rcl_lexer_state_t g_states[] = { T_TOKEN, 1, - RCL_TERMINAL_NONE, { {S8, 'a', 'z'}, {S8, 'A', 'Z'}, @@ -339,7 +331,6 @@ static const rcl_lexer_state_t g_states[] = { T_TOKEN, 1, - RCL_TERMINAL_NONE, { {S8, 'a', 'z'}, {S8, 'A', 'Z'}, @@ -351,7 +342,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S11, 'o', 'o'}, END_TRANSITIONS @@ -361,7 +351,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S12, 's', 's'}, END_TRANSITIONS @@ -371,7 +360,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S13, 't', 't'}, {S20, 's', 's'}, @@ -382,7 +370,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S14, 'o', 'o'}, END_TRANSITIONS @@ -392,7 +379,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S15, 'p', 'p'}, END_TRANSITIONS @@ -402,7 +388,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S16, 'i', 'i'}, END_TRANSITIONS @@ -412,7 +397,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S17, 'c', 'c'}, END_TRANSITIONS @@ -422,7 +406,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S18, ':', ':'}, END_TRANSITIONS @@ -432,7 +415,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 2, - RCL_TERMINAL_NONE, { {S19, '/', '/'}, END_TRANSITIONS @@ -442,7 +424,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 3, - RCL_TERMINAL_NONE, { {T_URL_TOPIC, '/', '/'}, END_TRANSITIONS @@ -452,7 +433,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S21, 'e', 'e'}, END_TRANSITIONS @@ -462,7 +442,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S22, 'r', 'r'}, END_TRANSITIONS @@ -472,7 +451,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S23, 'v', 'v'}, END_TRANSITIONS @@ -482,7 +460,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S24, 'i', 'i'}, END_TRANSITIONS @@ -492,7 +469,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S25, 'c', 'c'}, END_TRANSITIONS @@ -502,7 +478,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S26, 'e', 'e'}, END_TRANSITIONS @@ -512,7 +487,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 1, - RCL_TERMINAL_NONE, { {S27, ':', ':'}, END_TRANSITIONS @@ -522,7 +496,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 2, - RCL_TERMINAL_NONE, { {S28, '/', '/'}, END_TRANSITIONS @@ -532,7 +505,6 @@ static const rcl_lexer_state_t g_states[] = { S8, 3, - RCL_TERMINAL_NONE, { {T_URL_SERVICE, '/', '/'}, END_TRANSITIONS @@ -542,7 +514,6 @@ static const rcl_lexer_state_t g_states[] = { T_WILD_ONE, 1, - RCL_TERMINAL_NONE, { {T_WILD_MULTI, '*', '*'}, END_TRANSITIONS @@ -552,57 +523,60 @@ static const rcl_lexer_state_t g_states[] = { T_COLON, 1, - RCL_TERMINAL_NONE, { {T_SEPARATOR, '=', '='}, END_TRANSITIONS } }, - // Terminal states - // 31 - {S0, 0, RCL_TERMINAL_TILDE_SLASH, {END_TRANSITIONS}}, - // 32 - {S0, 0, RCL_TERMINAL_URL_SERVICE, {END_TRANSITIONS}}, - // 33 - {S0, 0, RCL_TERMINAL_URL_TOPIC, {END_TRANSITIONS}}, - // 34 - {S0, 0, RCL_TERMINAL_COLON, {END_TRANSITIONS}}, - // 35 - {S0, 0, RCL_TERMINAL_NODE, {END_TRANSITIONS}}, - // 36 - {S0, 0, RCL_TERMINAL_NS, {END_TRANSITIONS}}, - // 37 - {S0, 0, RCL_TERMINAL_SEPARATOR, {END_TRANSITIONS}}, - // 38 - {S0, 0, RCL_TERMINAL_BR1, {END_TRANSITIONS}}, - // 39 - {S0, 0, RCL_TERMINAL_BR2, {END_TRANSITIONS}}, - // 40 - {S0, 0, RCL_TERMINAL_BR3, {END_TRANSITIONS}}, - // 41 - {S0, 0, RCL_TERMINAL_BR4, {END_TRANSITIONS}}, - // 42 - {S0, 0, RCL_TERMINAL_BR5, {END_TRANSITIONS}}, - // 43 - {S0, 0, RCL_TERMINAL_BR6, {END_TRANSITIONS}}, - // 44 - {S0, 0, RCL_TERMINAL_BR7, {END_TRANSITIONS}}, - // 45 - {S0, 0, RCL_TERMINAL_BR8, {END_TRANSITIONS}}, - // 46 - {S0, 0, RCL_TERMINAL_BR9, {END_TRANSITIONS}}, - // 47 - {S0, 0, RCL_TERMINAL_TOKEN, {END_TRANSITIONS}}, - // 48 - {S0, 0, RCL_TERMINAL_FORWARD_SLASH, {END_TRANSITIONS}}, - // 49 - {S0, 0, RCL_TERMINAL_WILD_ONE, {END_TRANSITIONS}}, - // 50 - {S0, 0, RCL_TERMINAL_WILD_MULTI, {END_TRANSITIONS}}, - // 51 - {S0, 0, RCL_TERMINAL_EOF, {END_TRANSITIONS}}, - // 52 - {S0, 0, RCL_TERMINAL_NONE, {END_TRANSITIONS}}, +}; + +#include + +rcl_lexer_terminal_t g_terminals[LAST_TERMINAL + 1] = { + // 0 + RCL_TERMINAL_TILDE_SLASH, + // 1 + RCL_TERMINAL_URL_SERVICE, + // 2 + RCL_TERMINAL_URL_TOPIC, + // 3 + RCL_TERMINAL_COLON, + // 4 + RCL_TERMINAL_NODE, + // 5 + RCL_TERMINAL_NS, + // 6 + RCL_TERMINAL_SEPARATOR, + // 7 + RCL_TERMINAL_BR1, + // 8 + RCL_TERMINAL_BR2, + // 9 + RCL_TERMINAL_BR3, + // 10 + RCL_TERMINAL_BR4, + // 11 + RCL_TERMINAL_BR5, + // 12 + RCL_TERMINAL_BR6, + // 13 + RCL_TERMINAL_BR7, + // 14 + RCL_TERMINAL_BR8, + // 15 + RCL_TERMINAL_BR9, + // 16 + RCL_TERMINAL_TOKEN, + // 17 + RCL_TERMINAL_FORWARD_SLASH, + // 18 + RCL_TERMINAL_WILD_ONE, + // 19 + RCL_TERMINAL_WILD_MULTI, + // 20 + RCL_TERMINAL_EOF, + // 21 + RCL_TERMINAL_NONE, }; rcl_ret_t @@ -615,6 +589,7 @@ rcl_lexer_analyze( // RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, allocator); // RCL_CHECK_ARGUMENT_FOR_NULL(terminal, RCL_RET_INVALID_ARGUMENT, allocator); // RCL_CHECK_ARGUMENT_FOR_NULL(length, RCL_RET_INVALID_ARGUMENT, allocator); + printf("XXX sizeof state machine %lu\n", sizeof(g_states) + sizeof(g_terminals)); *length = 0; @@ -624,16 +599,17 @@ rcl_lexer_analyze( return RCL_RET_OK; } - const rcl_lexer_state_t * state = &(g_states[S0]); + const rcl_lexer_state_t * state; char current_char; - size_t next_state; + size_t next_state = S0; size_t movement; do { + state = &(g_states[next_state]); current_char = text[*length]; next_state = 0; movement = 0; - // Loop through all transitions in current state and find one that matches + // Look for a transition that contains this character in its range size_t transition_idx = 0; const rcl_lexer_transition_t * transition; do { @@ -645,24 +621,23 @@ rcl_lexer_analyze( ++transition_idx; } while (transition->to_state != 0); + // if no transition was found, take the else transition if (0 == next_state) { - // no transition found, take the else transition next_state = state->else_state; movement = state->else_movement; } + // Move the lexer to another character in the string if (0 == movement) { - // Advance position in string to test next char + // Go forwards 1 char ++(*length); } else { - // Go backwards in string + // Go backwards N chars *length -= movement - 1; // TODO(sloretz) Error if movement would cause length to overflow } - - state = &(g_states[next_state]); } while (next_state < FIRST_TERMINAL); - *terminal = state->terminal; + *terminal = g_terminals[next_state - FIRST_TERMINAL]; return RCL_RET_OK; } From 673cf3150e85d786f215516d80f3c2dda9bac235 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 11:21:23 -0700 Subject: [PATCH 06/38] Shrink lexer by using char, add error checking --- rcl/include/rcl/lexer.h | 5 +- rcl/src/rcl/lexer.c | 96 ++++++++++++++++++++----------------- rcl/test/rcl/test_lexer.cpp | 3 +- 3 files changed, 59 insertions(+), 45 deletions(-) diff --git a/rcl/include/rcl/lexer.h b/rcl/include/rcl/lexer.h index 855f21d71..196b096be 100644 --- a/rcl/include/rcl/lexer.h +++ b/rcl/include/rcl/lexer.h @@ -78,11 +78,14 @@ typedef enum rcl_lexer_terminal_t // Analize string until one terminal is found // If the string does not begin with a valid terminal, terminal will be RCL_TERMINAL_NONE // If the first character is '\0', the terminal will be RCL_TERMINAL_EOF +RCL_PUBLIC +RCL_WARN_UNUSED rcl_ret_t rcl_lexer_analyze( const char * text, rcl_lexer_terminal_t * terminal, - size_t * length); + size_t * length, + rcl_allocator_t allocator); #if __cplusplus } diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index ee32bb6e6..4452e9f28 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "rcl/error_handling.h" #include "rcl/lexer.h" /* The lexer tries to find a lexeme in a string. @@ -141,7 +142,7 @@ digraph remapping_lexer { typedef struct rcl_lexer_transition_t { // Index of a state to transition to - const size_t to_state; + const unsigned char to_state; // Start of a range of chars (inclusive) which activates this transition const char range_start; // End of a range of chars (inclusive) which activates this transition @@ -152,9 +153,9 @@ typedef struct rcl_lexer_transition_t typedef struct rcl_lexer_state_t { // If no transition matches this causes a character to be analyzed a second time in another state - const size_t else_state; + const unsigned char else_state; // Movement associated with taking else state - const size_t else_movement; + const unsigned char else_movement; // Transitions in the state machine (NULL value at end of array) const rcl_lexer_transition_t transitions[11]; } rcl_lexer_state_t; @@ -228,7 +229,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S0 { T_NONE, - 0, + 0u, { {T_FORWARD_SLASH, '/', '/'}, {S1, '\\', '\\'}, @@ -246,7 +247,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S1 { T_NONE, - 0, + 0u, { {T_BR1, '1', '1'}, {T_BR2, '2', '2'}, @@ -263,7 +264,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S2 { T_NONE, - 0, + 0u, { {T_TILDE_SLASH, '/', '/'}, END_TRANSITIONS @@ -272,7 +273,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S3 { S9, - 1, + 1u, { {S4, '_', '_'}, END_TRANSITIONS @@ -281,7 +282,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S4 { T_NONE, - 0, + 0u, { {S5, 'n', 'n'}, END_TRANSITIONS @@ -290,7 +291,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S5 { T_NONE, - 0, + 0u, { {T_NS, 's', 's'}, {S6, 'o', 'o'}, @@ -300,7 +301,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S6 { T_NONE, - 0, + 0u, { {S7, 'd', 'd'}, END_TRANSITIONS @@ -309,7 +310,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S7 { T_NONE, - 0, + 0u, { {T_NODE, 'e', 'e'}, END_TRANSITIONS @@ -318,7 +319,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S8 { T_TOKEN, - 1, + 1u, { {S8, 'a', 'z'}, {S8, 'A', 'Z'}, @@ -330,7 +331,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S9 { T_TOKEN, - 1, + 1u, { {S8, 'a', 'z'}, {S8, 'A', 'Z'}, @@ -341,7 +342,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S10 { S8, - 1, + 1u, { {S11, 'o', 'o'}, END_TRANSITIONS @@ -350,7 +351,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S11 { S8, - 1, + 1u, { {S12, 's', 's'}, END_TRANSITIONS @@ -359,7 +360,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S12 { S8, - 1, + 1u, { {S13, 't', 't'}, {S20, 's', 's'}, @@ -369,7 +370,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S13 { S8, - 1, + 1u, { {S14, 'o', 'o'}, END_TRANSITIONS @@ -378,7 +379,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S14 { S8, - 1, + 1u, { {S15, 'p', 'p'}, END_TRANSITIONS @@ -387,7 +388,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S15 { S8, - 1, + 1u, { {S16, 'i', 'i'}, END_TRANSITIONS @@ -396,7 +397,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S16 { S8, - 1, + 1u, { {S17, 'c', 'c'}, END_TRANSITIONS @@ -405,7 +406,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S17 { S8, - 1, + 1u, { {S18, ':', ':'}, END_TRANSITIONS @@ -414,7 +415,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S18 { S8, - 2, + 2u, { {S19, '/', '/'}, END_TRANSITIONS @@ -423,7 +424,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S19 { S8, - 3, + 3u, { {T_URL_TOPIC, '/', '/'}, END_TRANSITIONS @@ -432,7 +433,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S20 { S8, - 1, + 1u, { {S21, 'e', 'e'}, END_TRANSITIONS @@ -441,7 +442,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S21 { S8, - 1, + 1u, { {S22, 'r', 'r'}, END_TRANSITIONS @@ -450,7 +451,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S22 { S8, - 1, + 1u, { {S23, 'v', 'v'}, END_TRANSITIONS @@ -459,7 +460,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S23 { S8, - 1, + 1u, { {S24, 'i', 'i'}, END_TRANSITIONS @@ -468,7 +469,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S24 { S8, - 1, + 1u, { {S25, 'c', 'c'}, END_TRANSITIONS @@ -477,7 +478,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S25 { S8, - 1, + 1u, { {S26, 'e', 'e'}, END_TRANSITIONS @@ -486,7 +487,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S26 { S8, - 1, + 1u, { {S27, ':', ':'}, END_TRANSITIONS @@ -495,7 +496,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S27 { S8, - 2, + 2u, { {S28, '/', '/'}, END_TRANSITIONS @@ -504,7 +505,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S28 { S8, - 3, + 3u, { {T_URL_SERVICE, '/', '/'}, END_TRANSITIONS @@ -513,7 +514,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S29 { T_WILD_ONE, - 1, + 1u, { {T_WILD_MULTI, '*', '*'}, END_TRANSITIONS @@ -522,7 +523,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = // S30 { T_COLON, - 1, + 1u, { {T_SEPARATOR, '=', '='}, END_TRANSITIONS @@ -530,8 +531,6 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = }, }; -#include - rcl_lexer_terminal_t g_terminals[LAST_TERMINAL + 1] = { // 0 RCL_TERMINAL_TILDE_SLASH, @@ -583,13 +582,13 @@ rcl_ret_t rcl_lexer_analyze( const char * text, rcl_lexer_terminal_t * terminal, - size_t * length) + size_t * length, + rcl_allocator_t alloc) { - // TODO(sloretz) accept allocator just for error checking - // RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, allocator); - // RCL_CHECK_ARGUMENT_FOR_NULL(terminal, RCL_RET_INVALID_ARGUMENT, allocator); - // RCL_CHECK_ARGUMENT_FOR_NULL(length, RCL_RET_INVALID_ARGUMENT, allocator); - printf("XXX sizeof state machine %lu\n", sizeof(g_states) + sizeof(g_terminals)); + RCL_CHECK_ALLOCATOR_WITH_MSG(&alloc, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); + RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, alloc); + RCL_CHECK_ARGUMENT_FOR_NULL(terminal, RCL_RET_INVALID_ARGUMENT, alloc); + RCL_CHECK_ARGUMENT_FOR_NULL(length, RCL_RET_INVALID_ARGUMENT, alloc); *length = 0; @@ -604,6 +603,10 @@ rcl_lexer_analyze( size_t next_state = S0; size_t movement; do { + if (next_state > LAST_STATE) { + RCL_SET_ERROR_MSG("Internal lexer bug: next state does not exist", alloc); + return RCL_RET_ERROR; + } state = &(g_states[next_state]); current_char = text[*length]; next_state = 0; @@ -633,11 +636,18 @@ rcl_lexer_analyze( ++(*length); } else { // Go backwards N chars + if (movement - 1 > *length) { + RCL_SET_ERROR_MSG("Internal lexer bug: movement would read before start of string", alloc); + return RCL_RET_ERROR; + } *length -= movement - 1; - // TODO(sloretz) Error if movement would cause length to overflow } } while (next_state < FIRST_TERMINAL); + if (FIRST_TERMINAL > next_state || next_state - FIRST_TERMINAL > LAST_TERMINAL) { + RCL_SET_ERROR_MSG("Internal lexer bug: terminal state does not exist", alloc); + return RCL_RET_ERROR; + } *terminal = g_terminals[next_state - FIRST_TERMINAL]; return RCL_RET_OK; } diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index a5d5de61a..0a1ad647d 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -43,7 +43,8 @@ class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test do { \ rcl_lexer_terminal_t actual_terminal; \ size_t end_pos; \ - rcl_ret_t ret = rcl_lexer_analyze(text, &actual_terminal, &end_pos); \ + rcl_allocator_t allocator = rcl_get_default_allocator(); \ + rcl_ret_t ret = rcl_lexer_analyze(text, &actual_terminal, &end_pos, allocator); \ ASSERT_EQ(RCL_RET_OK, ret); \ EXPECT_EQ(expected_terminal, actual_terminal); \ std::string actual_text(text, end_pos); \ From 7b220790238f6dab1b47c01a0c04a7ab939c8f35 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 11:25:20 -0700 Subject: [PATCH 07/38] Comment/whitespace --- rcl/src/rcl/lexer.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index 4452e9f28..ffc47ff45 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -31,9 +31,6 @@ * The movement M is written as M = 1 - N so it can be stored in an unsigned integer. * For example, an `` transition with M = 0 moves the lexer forwards 1 character, M = 1 keeps * the lexer at the current character, and M = 2 moves the lexer backwards one character. - * - * - * dot graph of state machine digraph remapping_lexer { rankdir=LR; @@ -137,7 +134,6 @@ digraph remapping_lexer { } */ - // Represents a transition from one state to another typedef struct rcl_lexer_transition_t { @@ -225,7 +221,6 @@ typedef struct rcl_lexer_state_t static const rcl_lexer_state_t g_states[LAST_STATE + 1] = { - // Nonterminal states // S0 { T_NONE, @@ -602,6 +597,8 @@ rcl_lexer_analyze( char current_char; size_t next_state = S0; size_t movement; + + // Analyze one character at a time until lexeme is found do { if (next_state > LAST_STATE) { RCL_SET_ERROR_MSG("Internal lexer bug: next state does not exist", alloc); From e9ac9bf8628fb30254a00262b316efe612ec9e68 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 11:26:10 -0700 Subject: [PATCH 08/38] comment --- rcl/test/rcl/test_lexer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index 0a1ad647d..3fca135f7 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -38,7 +38,7 @@ class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test } }; -// Not using a function because with a macro gtest shows the line number where the macro is used +// Not using a function so gtest failure output shows the line number where the macro is used #define EXPECT_LEX(expected_terminal, expected_text, text) \ do { \ rcl_lexer_terminal_t actual_terminal; \ From c28589ef987895013cba017af23c17a8c6a52b58 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 11:31:39 -0700 Subject: [PATCH 09/38] terminal -> lexeme where appropriate --- rcl/include/rcl/lexer.h | 58 ++--- rcl/src/rcl/lexer.c | 56 ++--- rcl/test/rcl/test_lexer.cpp | 410 ++++++++++++++++++------------------ 3 files changed, 262 insertions(+), 262 deletions(-) diff --git a/rcl/include/rcl/lexer.h b/rcl/include/rcl/lexer.h index 196b096be..e25997fa1 100644 --- a/rcl/include/rcl/lexer.h +++ b/rcl/include/rcl/lexer.h @@ -27,63 +27,63 @@ extern "C" { #endif -typedef enum rcl_lexer_terminal_t +typedef enum rcl_lexeme_t { - // Indicates no valid terminal was found - RCL_TERMINAL_NONE = 0, + // Indicates no valid lexeme was found + RCL_LEXEME_NONE = 0, // Indicates end of input has been reached - RCL_TERMINAL_EOF = 1, + RCL_LEXEME_EOF = 1, // ~/ - RCL_TERMINAL_TILDE_SLASH = 2, + RCL_LEXEME_TILDE_SLASH = 2, // rosservice:// - RCL_TERMINAL_URL_SERVICE = 3, + RCL_LEXEME_URL_SERVICE = 3, // rostopic:// - RCL_TERMINAL_URL_TOPIC = 4, + RCL_LEXEME_URL_TOPIC = 4, // : - RCL_TERMINAL_COLON = 5, + RCL_LEXEME_COLON = 5, // __node - RCL_TERMINAL_NODE = 6, + RCL_LEXEME_NODE = 6, // __ns - RCL_TERMINAL_NS = 7, + RCL_LEXEME_NS = 7, // := - RCL_TERMINAL_SEPARATOR = 8, + RCL_LEXEME_SEPARATOR = 8, // \1 - RCL_TERMINAL_BR1 = 9, + RCL_LEXEME_BR1 = 9, // \2 - RCL_TERMINAL_BR2 = 10, + RCL_LEXEME_BR2 = 10, // \3 - RCL_TERMINAL_BR3 = 11, + RCL_LEXEME_BR3 = 11, // \4 - RCL_TERMINAL_BR4 = 12, + RCL_LEXEME_BR4 = 12, // \5 - RCL_TERMINAL_BR5 = 13, + RCL_LEXEME_BR5 = 13, // \6 - RCL_TERMINAL_BR6 = 14, + RCL_LEXEME_BR6 = 14, // \7 - RCL_TERMINAL_BR7 = 15, + RCL_LEXEME_BR7 = 15, // \8 - RCL_TERMINAL_BR8 = 16, + RCL_LEXEME_BR8 = 16, // \9 - RCL_TERMINAL_BR9 = 17, + RCL_LEXEME_BR9 = 17, // a name between slashes, must match (([a-zA-Z](_)?)|_)([0-9a-zA-Z](_)?)* - RCL_TERMINAL_TOKEN = 18, + RCL_LEXEME_TOKEN = 18, // / - RCL_TERMINAL_FORWARD_SLASH = 19, + RCL_LEXEME_FORWARD_SLASH = 19, // * - RCL_TERMINAL_WILD_ONE = 20, + RCL_LEXEME_WILD_ONE = 20, // ** - RCL_TERMINAL_WILD_MULTI = 21 -} rcl_lexer_terminal_t; + RCL_LEXEME_WILD_MULTI = 21 +} rcl_lexeme_t; -// Analize string until one terminal is found -// If the string does not begin with a valid terminal, terminal will be RCL_TERMINAL_NONE -// If the first character is '\0', the terminal will be RCL_TERMINAL_EOF +// Analize string until one lexeme is found +// If the string does not begin with a valid lexeme, lexeme will be RCL_LEXEME_NONE +// If the first character is '\0', the lexeme will be RCL_LEXEME_EOF RCL_PUBLIC RCL_WARN_UNUSED rcl_ret_t rcl_lexer_analyze( const char * text, - rcl_lexer_terminal_t * terminal, + rcl_lexeme_t * lexeme, size_t * length, rcl_allocator_t allocator); diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index ffc47ff45..3e7494342 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -145,7 +145,7 @@ typedef struct rcl_lexer_transition_t const char range_end; } rcl_lexer_transition_t; -// Represents a state that either has transitions or is terminal +// Represents a non-terminal state typedef struct rcl_lexer_state_t { // If no transition matches this causes a character to be analyzed a second time in another state @@ -526,70 +526,70 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = }, }; -rcl_lexer_terminal_t g_terminals[LAST_TERMINAL + 1] = { +rcl_lexeme_t g_terminals[LAST_TERMINAL + 1] = { // 0 - RCL_TERMINAL_TILDE_SLASH, + RCL_LEXEME_TILDE_SLASH, // 1 - RCL_TERMINAL_URL_SERVICE, + RCL_LEXEME_URL_SERVICE, // 2 - RCL_TERMINAL_URL_TOPIC, + RCL_LEXEME_URL_TOPIC, // 3 - RCL_TERMINAL_COLON, + RCL_LEXEME_COLON, // 4 - RCL_TERMINAL_NODE, + RCL_LEXEME_NODE, // 5 - RCL_TERMINAL_NS, + RCL_LEXEME_NS, // 6 - RCL_TERMINAL_SEPARATOR, + RCL_LEXEME_SEPARATOR, // 7 - RCL_TERMINAL_BR1, + RCL_LEXEME_BR1, // 8 - RCL_TERMINAL_BR2, + RCL_LEXEME_BR2, // 9 - RCL_TERMINAL_BR3, + RCL_LEXEME_BR3, // 10 - RCL_TERMINAL_BR4, + RCL_LEXEME_BR4, // 11 - RCL_TERMINAL_BR5, + RCL_LEXEME_BR5, // 12 - RCL_TERMINAL_BR6, + RCL_LEXEME_BR6, // 13 - RCL_TERMINAL_BR7, + RCL_LEXEME_BR7, // 14 - RCL_TERMINAL_BR8, + RCL_LEXEME_BR8, // 15 - RCL_TERMINAL_BR9, + RCL_LEXEME_BR9, // 16 - RCL_TERMINAL_TOKEN, + RCL_LEXEME_TOKEN, // 17 - RCL_TERMINAL_FORWARD_SLASH, + RCL_LEXEME_FORWARD_SLASH, // 18 - RCL_TERMINAL_WILD_ONE, + RCL_LEXEME_WILD_ONE, // 19 - RCL_TERMINAL_WILD_MULTI, + RCL_LEXEME_WILD_MULTI, // 20 - RCL_TERMINAL_EOF, + RCL_LEXEME_EOF, // 21 - RCL_TERMINAL_NONE, + RCL_LEXEME_NONE, }; rcl_ret_t rcl_lexer_analyze( const char * text, - rcl_lexer_terminal_t * terminal, + rcl_lexeme_t * lexeme, size_t * length, rcl_allocator_t alloc) { RCL_CHECK_ALLOCATOR_WITH_MSG(&alloc, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, alloc); - RCL_CHECK_ARGUMENT_FOR_NULL(terminal, RCL_RET_INVALID_ARGUMENT, alloc); + RCL_CHECK_ARGUMENT_FOR_NULL(lexeme, RCL_RET_INVALID_ARGUMENT, alloc); RCL_CHECK_ARGUMENT_FOR_NULL(length, RCL_RET_INVALID_ARGUMENT, alloc); *length = 0; if ('\0' == text[0]) { // Early exit if string is empty - *terminal = RCL_TERMINAL_EOF; + *lexeme = RCL_LEXEME_EOF; return RCL_RET_OK; } @@ -645,6 +645,6 @@ rcl_lexer_analyze( RCL_SET_ERROR_MSG("Internal lexer bug: terminal state does not exist", alloc); return RCL_RET_ERROR; } - *terminal = g_terminals[next_state - FIRST_TERMINAL]; + *lexeme = g_terminals[next_state - FIRST_TERMINAL]; return RCL_RET_OK; } diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index 3fca135f7..7e7110e70 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -39,14 +39,14 @@ class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test }; // Not using a function so gtest failure output shows the line number where the macro is used -#define EXPECT_LEX(expected_terminal, expected_text, text) \ +#define EXPECT_LEX(expected_lexeme, expected_text, text) \ do { \ - rcl_lexer_terminal_t actual_terminal; \ + rcl_lexeme_t actual_lexeme; \ size_t end_pos; \ rcl_allocator_t allocator = rcl_get_default_allocator(); \ - rcl_ret_t ret = rcl_lexer_analyze(text, &actual_terminal, &end_pos, allocator); \ + rcl_ret_t ret = rcl_lexer_analyze(text, &actual_lexeme, &end_pos, allocator); \ ASSERT_EQ(RCL_RET_OK, ret); \ - EXPECT_EQ(expected_terminal, actual_terminal); \ + EXPECT_EQ(expected_lexeme, actual_lexeme); \ std::string actual_text(text, end_pos); \ EXPECT_STREQ(expected_text, actual_text.c_str()); \ } while (false) @@ -54,276 +54,276 @@ class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_token) { // Things get recognized as tokens whether input ends or non token characters come after them - EXPECT_LEX(RCL_TERMINAL_TOKEN, "foo", "foo"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "foo", "foo:"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "foo", "foo"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "foo", "foo:"); // Check full range for starting character - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a", "a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "b", "b"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "c", "c"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "d", "d"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "e", "e"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "f", "f"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "g", "g"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "h", "h"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "i", "i"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "j", "j"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "k", "k"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "l", "l"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "m", "m"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "n", "n"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "o", "o"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "p", "p"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "q", "q"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "r", "r"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "s", "s"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "t", "t"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "u", "u"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "v", "v"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "w", "w"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "x", "x"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "y", "y"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "z", "z"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "A", "A"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "B", "B"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "C", "C"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "D", "D"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "E", "E"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "F", "F"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "G", "G"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "H", "H"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "I", "I"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "J", "J"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "K", "K"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "L", "L"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "M", "M"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "N", "N"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "O", "O"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "P", "P"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "Q", "Q"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "R", "R"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "S", "S"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "T", "T"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "U", "U"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "V", "V"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "W", "W"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "X", "X"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "Y", "Y"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "Z", "Z"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_", "_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a", "a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "b", "b"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "c", "c"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "d", "d"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "e", "e"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "f", "f"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "g", "g"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "h", "h"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "i", "i"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "j", "j"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "k", "k"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "l", "l"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "m", "m"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "n", "n"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "o", "o"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "p", "p"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "q", "q"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "r", "r"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "s", "s"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "t", "t"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "u", "u"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "v", "v"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "w", "w"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "x", "x"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "y", "y"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "z", "z"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "A", "A"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "B", "B"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "C", "C"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "D", "D"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "E", "E"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "F", "F"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "G", "G"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "H", "H"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "I", "I"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "J", "J"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "K", "K"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "L", "L"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "M", "M"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "N", "N"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "O", "O"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "P", "P"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "Q", "Q"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "R", "R"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "S", "S"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "T", "T"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "U", "U"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "V", "V"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "W", "W"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "X", "X"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "Y", "Y"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "Z", "Z"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_", "_"); // Check banned characters adjacent to allowed ones in ASCII - EXPECT_LEX(RCL_TERMINAL_NONE, "@", "@"); - EXPECT_LEX(RCL_TERMINAL_NONE, "[", "["); - EXPECT_LEX(RCL_TERMINAL_NONE, "`", "`"); - EXPECT_LEX(RCL_TERMINAL_NONE, "{", "{"); + EXPECT_LEX(RCL_LEXEME_NONE, "@", "@"); + EXPECT_LEX(RCL_LEXEME_NONE, "[", "["); + EXPECT_LEX(RCL_LEXEME_NONE, "`", "`"); + EXPECT_LEX(RCL_LEXEME_NONE, "{", "{"); // Tokens cannot start with digits - EXPECT_LEX(RCL_TERMINAL_NONE, "0", "0"); - EXPECT_LEX(RCL_TERMINAL_NONE, "1", "1"); - EXPECT_LEX(RCL_TERMINAL_NONE, "2", "2"); - EXPECT_LEX(RCL_TERMINAL_NONE, "3", "3"); - EXPECT_LEX(RCL_TERMINAL_NONE, "4", "4"); - EXPECT_LEX(RCL_TERMINAL_NONE, "5", "5"); - EXPECT_LEX(RCL_TERMINAL_NONE, "6", "6"); - EXPECT_LEX(RCL_TERMINAL_NONE, "7", "7"); - EXPECT_LEX(RCL_TERMINAL_NONE, "8", "8"); - EXPECT_LEX(RCL_TERMINAL_NONE, "9", "9"); + EXPECT_LEX(RCL_LEXEME_NONE, "0", "0"); + EXPECT_LEX(RCL_LEXEME_NONE, "1", "1"); + EXPECT_LEX(RCL_LEXEME_NONE, "2", "2"); + EXPECT_LEX(RCL_LEXEME_NONE, "3", "3"); + EXPECT_LEX(RCL_LEXEME_NONE, "4", "4"); + EXPECT_LEX(RCL_LEXEME_NONE, "5", "5"); + EXPECT_LEX(RCL_LEXEME_NONE, "6", "6"); + EXPECT_LEX(RCL_LEXEME_NONE, "7", "7"); + EXPECT_LEX(RCL_LEXEME_NONE, "8", "8"); + EXPECT_LEX(RCL_LEXEME_NONE, "9", "9"); // Tokens may contain underscores - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_abcd", "_abcd"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "abcd_", "abcd_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "ab_cd", "ab_cd"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_a_b_c_d_", "_a_b_c_d_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_abcd", "_abcd"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "abcd_", "abcd_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "ab_cd", "ab_cd"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_a_b_c_d_", "_a_b_c_d_"); // Tokens cannot contain double underscores - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_a_", "_a__bcd"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a_", "a__bcd"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "A_", "A__bcd"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__a", "__a"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__A", "__A"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_a_", "_a__bcd"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a_", "a__bcd"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "A_", "A__bcd"); + EXPECT_LEX(RCL_LEXEME_NONE, "__a", "__a"); + EXPECT_LEX(RCL_LEXEME_NONE, "__A", "__A"); // Tokens may contain digits - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_0_", "_0_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_1_", "_1_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_2_", "_2_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_3_", "_3_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_4_", "_4_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_5_", "_5_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_6_", "_6_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_7_", "_7_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_8_", "_8_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_9_", "_9_"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a0a", "a0a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a1a", "a1a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a2a", "a2a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a3a", "a3a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a4a", "a4a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a5a", "a5a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a6a", "a6a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a7a", "a7a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a8a", "a8a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a9a", "a9a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_0_", "_0_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_1_", "_1_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_2_", "_2_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_3_", "_3_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_4_", "_4_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_5_", "_5_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_6_", "_6_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_7_", "_7_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_8_", "_8_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_9_", "_9_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a0a", "a0a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a1a", "a1a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a2a", "a2a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a3a", "a3a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a4a", "a4a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a5a", "a5a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a6a", "a6a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a7a", "a7a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a8a", "a8a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a9a", "a9a"); // Tokens may end with digits - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_0", "_0"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_1", "_1"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_2", "_2"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_3", "_3"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_4", "_4"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_5", "_5"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_6", "_6"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_7", "_7"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_8", "_8"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_9", "_9"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a0", "a0"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a1", "a1"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a2", "a2"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a3", "a3"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a4", "a4"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a5", "a5"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a6", "a6"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a7", "a7"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a8", "a8"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "a9", "a9"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_0", "_0"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_1", "_1"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_2", "_2"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_3", "_3"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_4", "_4"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_5", "_5"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_6", "_6"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_7", "_7"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_8", "_8"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_9", "_9"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a0", "a0"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a1", "a1"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a2", "a2"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a3", "a3"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a4", "a4"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a5", "a5"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a6", "a6"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a7", "a7"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a8", "a8"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "a9", "a9"); // Things that almost look like a url scheme but are actually tokens - EXPECT_LEX(RCL_TERMINAL_TOKEN, "ro", "ro"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "ros", "ros"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "ross", "ross"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosse", "rosse"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosser", "rosser"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosserv", "rosserv"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservi", "rosservi"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservic", "rosservic"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:="); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:/"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosservice", "rosservice:/a"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rost", "rost"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rosto", "rosto"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostop", "rostop"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopi", "rostopi"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:="); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:/"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "rostopic", "rostopic:/a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "ro", "ro"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "ros", "ros"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "ross", "ross"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosse", "rosse"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosser", "rosser"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosserv", "rosserv"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosservi", "rosservi"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosservic", "rosservic"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosservice", "rosservice"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosservice", "rosservice:"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosservice", "rosservice:="); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosservice", "rosservice:/"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosservice", "rosservice:/a"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rost", "rost"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rosto", "rosto"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rostop", "rostop"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rostopi", "rostopi"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rostopic", "rostopic"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rostopic", "rostopic:"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rostopic", "rostopic:="); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rostopic", "rostopic:/"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "rostopic", "rostopic:/a"); // Tokens may contain uppercase characters - EXPECT_LEX(RCL_TERMINAL_TOKEN, "ABC", "ABC"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_DEF", "_DEF"); - EXPECT_LEX(RCL_TERMINAL_TOKEN, "_GHI_", "_GHI_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "ABC", "ABC"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_DEF", "_DEF"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "_GHI_", "_GHI_"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_url_scheme) { // No text after scheme - EXPECT_LEX(RCL_TERMINAL_URL_SERVICE, "rosservice://", "rosservice://"); - EXPECT_LEX(RCL_TERMINAL_URL_TOPIC, "rostopic://", "rostopic://"); + EXPECT_LEX(RCL_LEXEME_URL_SERVICE, "rosservice://", "rosservice://"); + EXPECT_LEX(RCL_LEXEME_URL_TOPIC, "rostopic://", "rostopic://"); // Some text after scheme - EXPECT_LEX(RCL_TERMINAL_URL_SERVICE, "rosservice://", "rosservice://abcd"); - EXPECT_LEX(RCL_TERMINAL_URL_SERVICE, "rosservice://", "rosservice:///"); - EXPECT_LEX(RCL_TERMINAL_URL_TOPIC, "rostopic://", "rostopic://abcd"); - EXPECT_LEX(RCL_TERMINAL_URL_TOPIC, "rostopic://", "rostopic:///"); + EXPECT_LEX(RCL_LEXEME_URL_SERVICE, "rosservice://", "rosservice://abcd"); + EXPECT_LEX(RCL_LEXEME_URL_SERVICE, "rosservice://", "rosservice:///"); + EXPECT_LEX(RCL_LEXEME_URL_TOPIC, "rostopic://", "rostopic://abcd"); + EXPECT_LEX(RCL_LEXEME_URL_TOPIC, "rostopic://", "rostopic:///"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_backreferences) { // No text after backreference - EXPECT_LEX(RCL_TERMINAL_BR1, "\\1", "\\1"); - EXPECT_LEX(RCL_TERMINAL_BR2, "\\2", "\\2"); - EXPECT_LEX(RCL_TERMINAL_BR3, "\\3", "\\3"); - EXPECT_LEX(RCL_TERMINAL_BR4, "\\4", "\\4"); - EXPECT_LEX(RCL_TERMINAL_BR5, "\\5", "\\5"); - EXPECT_LEX(RCL_TERMINAL_BR6, "\\6", "\\6"); - EXPECT_LEX(RCL_TERMINAL_BR7, "\\7", "\\7"); - EXPECT_LEX(RCL_TERMINAL_BR8, "\\8", "\\8"); - EXPECT_LEX(RCL_TERMINAL_BR9, "\\9", "\\9"); + EXPECT_LEX(RCL_LEXEME_BR1, "\\1", "\\1"); + EXPECT_LEX(RCL_LEXEME_BR2, "\\2", "\\2"); + EXPECT_LEX(RCL_LEXEME_BR3, "\\3", "\\3"); + EXPECT_LEX(RCL_LEXEME_BR4, "\\4", "\\4"); + EXPECT_LEX(RCL_LEXEME_BR5, "\\5", "\\5"); + EXPECT_LEX(RCL_LEXEME_BR6, "\\6", "\\6"); + EXPECT_LEX(RCL_LEXEME_BR7, "\\7", "\\7"); + EXPECT_LEX(RCL_LEXEME_BR8, "\\8", "\\8"); + EXPECT_LEX(RCL_LEXEME_BR9, "\\9", "\\9"); // Some text after backreference - EXPECT_LEX(RCL_TERMINAL_BR1, "\\1", "\\1a"); - EXPECT_LEX(RCL_TERMINAL_BR2, "\\2", "\\2a"); - EXPECT_LEX(RCL_TERMINAL_BR3, "\\3", "\\3a"); - EXPECT_LEX(RCL_TERMINAL_BR4, "\\4", "\\4a"); - EXPECT_LEX(RCL_TERMINAL_BR5, "\\5", "\\5a"); - EXPECT_LEX(RCL_TERMINAL_BR6, "\\6", "\\6a"); - EXPECT_LEX(RCL_TERMINAL_BR7, "\\7", "\\7a"); - EXPECT_LEX(RCL_TERMINAL_BR8, "\\8", "\\8a"); - EXPECT_LEX(RCL_TERMINAL_BR9, "\\9", "\\9a"); + EXPECT_LEX(RCL_LEXEME_BR1, "\\1", "\\1a"); + EXPECT_LEX(RCL_LEXEME_BR2, "\\2", "\\2a"); + EXPECT_LEX(RCL_LEXEME_BR3, "\\3", "\\3a"); + EXPECT_LEX(RCL_LEXEME_BR4, "\\4", "\\4a"); + EXPECT_LEX(RCL_LEXEME_BR5, "\\5", "\\5a"); + EXPECT_LEX(RCL_LEXEME_BR6, "\\6", "\\6a"); + EXPECT_LEX(RCL_LEXEME_BR7, "\\7", "\\7a"); + EXPECT_LEX(RCL_LEXEME_BR8, "\\8", "\\8a"); + EXPECT_LEX(RCL_LEXEME_BR9, "\\9", "\\9a"); // Not valid backreferences - EXPECT_LEX(RCL_TERMINAL_NONE, "\\0", "\\0"); - EXPECT_LEX(RCL_TERMINAL_NONE, "\\a", "\\a"); - EXPECT_LEX(RCL_TERMINAL_NONE, "\\Z", "\\Z"); - EXPECT_LEX(RCL_TERMINAL_NONE, "\\_", "\\_"); + EXPECT_LEX(RCL_LEXEME_NONE, "\\0", "\\0"); + EXPECT_LEX(RCL_LEXEME_NONE, "\\a", "\\a"); + EXPECT_LEX(RCL_LEXEME_NONE, "\\Z", "\\Z"); + EXPECT_LEX(RCL_LEXEME_NONE, "\\_", "\\_"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_forward_slash) { - EXPECT_LEX(RCL_TERMINAL_FORWARD_SLASH, "/", "/"); - EXPECT_LEX(RCL_TERMINAL_FORWARD_SLASH, "/", "//"); - EXPECT_LEX(RCL_TERMINAL_FORWARD_SLASH, "/", "/_"); + EXPECT_LEX(RCL_LEXEME_FORWARD_SLASH, "/", "/"); + EXPECT_LEX(RCL_LEXEME_FORWARD_SLASH, "/", "//"); + EXPECT_LEX(RCL_LEXEME_FORWARD_SLASH, "/", "/_"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_wildcards) { - EXPECT_LEX(RCL_TERMINAL_WILD_ONE, "*", "*"); - EXPECT_LEX(RCL_TERMINAL_WILD_ONE, "*", "*/"); - EXPECT_LEX(RCL_TERMINAL_WILD_MULTI, "**", "**"); - EXPECT_LEX(RCL_TERMINAL_WILD_MULTI, "**", "**/"); + EXPECT_LEX(RCL_LEXEME_WILD_ONE, "*", "*"); + EXPECT_LEX(RCL_LEXEME_WILD_ONE, "*", "*/"); + EXPECT_LEX(RCL_LEXEME_WILD_MULTI, "**", "**"); + EXPECT_LEX(RCL_LEXEME_WILD_MULTI, "**", "**/"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_colon) { - EXPECT_LEX(RCL_TERMINAL_COLON, ":", ":"); - EXPECT_LEX(RCL_TERMINAL_COLON, ":", ":r"); + EXPECT_LEX(RCL_LEXEME_COLON, ":", ":"); + EXPECT_LEX(RCL_LEXEME_COLON, ":", ":r"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_separator) { - EXPECT_LEX(RCL_TERMINAL_SEPARATOR, ":=", ":="); - EXPECT_LEX(RCL_TERMINAL_SEPARATOR, ":=", ":=0"); + EXPECT_LEX(RCL_LEXEME_SEPARATOR, ":=", ":="); + EXPECT_LEX(RCL_LEXEME_SEPARATOR, ":=", ":=0"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_ns) { // Has __ns - EXPECT_LEX(RCL_TERMINAL_NS, "__ns", "__ns"); - EXPECT_LEX(RCL_TERMINAL_NS, "__ns", "__nsssss"); + EXPECT_LEX(RCL_LEXEME_NS, "__ns", "__ns"); + EXPECT_LEX(RCL_LEXEME_NS, "__ns", "__nsssss"); // Things that are almost __ns - EXPECT_LEX(RCL_TERMINAL_NONE, "__", "__"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__n", "__n"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__n!", "__n!"); + EXPECT_LEX(RCL_LEXEME_NONE, "__", "__"); + EXPECT_LEX(RCL_LEXEME_NONE, "__n", "__n"); + EXPECT_LEX(RCL_LEXEME_NONE, "__n!", "__n!"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_node) { // Has __node - EXPECT_LEX(RCL_TERMINAL_NODE, "__node", "__node"); - EXPECT_LEX(RCL_TERMINAL_NODE, "__node", "__nodessss"); + EXPECT_LEX(RCL_LEXEME_NODE, "__node", "__node"); + EXPECT_LEX(RCL_LEXEME_NODE, "__node", "__nodessss"); // Things that are almost __node - EXPECT_LEX(RCL_TERMINAL_NONE, "__", "__"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__n", "__n"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__na", "__na"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__no", "__no"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__noa", "__noa"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__nod", "__nod"); - EXPECT_LEX(RCL_TERMINAL_NONE, "__noda", "__noda"); + EXPECT_LEX(RCL_LEXEME_NONE, "__", "__"); + EXPECT_LEX(RCL_LEXEME_NONE, "__n", "__n"); + EXPECT_LEX(RCL_LEXEME_NONE, "__na", "__na"); + EXPECT_LEX(RCL_LEXEME_NONE, "__no", "__no"); + EXPECT_LEX(RCL_LEXEME_NONE, "__noa", "__noa"); + EXPECT_LEX(RCL_LEXEME_NONE, "__nod", "__nod"); + EXPECT_LEX(RCL_LEXEME_NONE, "__noda", "__noda"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_tilde_slash) { - EXPECT_LEX(RCL_TERMINAL_TILDE_SLASH, "~/", "~/"); - EXPECT_LEX(RCL_TERMINAL_TILDE_SLASH, "~/", "~//"); - EXPECT_LEX(RCL_TERMINAL_NONE, "~", "~"); - EXPECT_LEX(RCL_TERMINAL_NONE, "~!", "~!"); + EXPECT_LEX(RCL_LEXEME_TILDE_SLASH, "~/", "~/"); + EXPECT_LEX(RCL_LEXEME_TILDE_SLASH, "~/", "~//"); + EXPECT_LEX(RCL_LEXEME_NONE, "~", "~"); + EXPECT_LEX(RCL_LEXEME_NONE, "~!", "~!"); } TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_eof) { - EXPECT_LEX(RCL_TERMINAL_EOF, "", ""); + EXPECT_LEX(RCL_LEXEME_EOF, "", ""); } From 975f2708bfa6b49edc4e364dad3e4eb148381ca6 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 11:35:12 -0700 Subject: [PATCH 10/38] Static const global --- rcl/src/rcl/lexer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index 3e7494342..1e93d6522 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -526,7 +526,7 @@ static const rcl_lexer_state_t g_states[LAST_STATE + 1] = }, }; -rcl_lexeme_t g_terminals[LAST_TERMINAL + 1] = { +static const rcl_lexeme_t g_terminals[LAST_TERMINAL + 1] = { // 0 RCL_LEXEME_TILDE_SLASH, // 1 From 4b9cb4f1873cf731732428994fa5e452b467c6f0 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 11:48:48 -0700 Subject: [PATCH 11/38] Documentation and argument order --- rcl/include/rcl/lexer.h | 33 ++++++++++++++++++++++++++++----- rcl/src/rcl/lexer.c | 4 ++-- rcl/test/rcl/test_lexer.cpp | 2 +- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/rcl/include/rcl/lexer.h b/rcl/include/rcl/lexer.h index e25997fa1..5c0d77c01 100644 --- a/rcl/include/rcl/lexer.h +++ b/rcl/include/rcl/lexer.h @@ -75,17 +75,40 @@ typedef enum rcl_lexeme_t RCL_LEXEME_WILD_MULTI = 21 } rcl_lexeme_t; -// Analize string until one lexeme is found -// If the string does not begin with a valid lexeme, lexeme will be RCL_LEXEME_NONE -// If the first character is '\0', the lexeme will be RCL_LEXEME_EOF + +/// Do lexical analysis on a string. +/** + * This function analyzes a string to see if it starts with a valid lexeme. + * If the string does not begin with a valid lexeme then lexeme will be RCL_LEXEME_NONE, and the + * length will be set to include the character that made it impossible. + * If the first character is '\0' then lexeme will be RCL_LEXEME_EOF. + * + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | Yes [1] + * Thread-Safe | Yes + * Uses Atomics | No + * Lock-Free | Yes + * [1] Only allocates if an argument is invalid or an internal bug is detected. + * + * \param[in] text The string to analyze. + * \param[in] allocator An allocator to use if an error occurs. + * \param[out] lexeme The type of lexeme found in the string. + * \param[out] length The length of text in the string that constitutes the found lexeme. + * \return `RCL_RET_OK` if analysis is successful regardless whether a valid lexeme is found, or + * \return `RCL_RET_INVALID_ARGUMENT` if any function arguments are invalid, or + * \return `RCL_RET_BAD_ALLOC` if allocating memory failed, or + * \return `RCL_RET_ERROR` if an internal bug is detected. + */ RCL_PUBLIC RCL_WARN_UNUSED rcl_ret_t rcl_lexer_analyze( const char * text, + rcl_allocator_t allocator, rcl_lexeme_t * lexeme, - size_t * length, - rcl_allocator_t allocator); + size_t * length); #if __cplusplus } diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index 1e93d6522..0664a5900 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -576,9 +576,9 @@ static const rcl_lexeme_t g_terminals[LAST_TERMINAL + 1] = { rcl_ret_t rcl_lexer_analyze( const char * text, + rcl_allocator_t alloc, rcl_lexeme_t * lexeme, - size_t * length, - rcl_allocator_t alloc) + size_t * length) { RCL_CHECK_ALLOCATOR_WITH_MSG(&alloc, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, alloc); diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index 7e7110e70..cb59c6bc5 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -44,7 +44,7 @@ class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test rcl_lexeme_t actual_lexeme; \ size_t end_pos; \ rcl_allocator_t allocator = rcl_get_default_allocator(); \ - rcl_ret_t ret = rcl_lexer_analyze(text, &actual_lexeme, &end_pos, allocator); \ + rcl_ret_t ret = rcl_lexer_analyze(text, allocator, &actual_lexeme, &end_pos); \ ASSERT_EQ(RCL_RET_OK, ret); \ EXPECT_EQ(expected_lexeme, actual_lexeme); \ std::string actual_text(text, end_pos); \ From 20a04577a9317db74b2a07c040dcc3e7a819da62 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 16:33:23 -0700 Subject: [PATCH 12/38] Add rcl_lexer_lookahead2_t --- rcl/CMakeLists.txt | 1 + rcl/include/rcl/lexer_lookahead.h | 240 +++++++++++++++++++++ rcl/include/rcl/types.h | 2 + rcl/src/rcl/lexer_lookahead.c | 228 ++++++++++++++++++++ rcl/test/CMakeLists.txt | 8 + rcl/test/rcl/test_lexer_lookahead.cpp | 286 ++++++++++++++++++++++++++ 6 files changed, 765 insertions(+) create mode 100644 rcl/include/rcl/lexer_lookahead.h create mode 100644 rcl/src/rcl/lexer_lookahead.c create mode 100644 rcl/test/rcl/test_lexer_lookahead.cpp diff --git a/rcl/CMakeLists.txt b/rcl/CMakeLists.txt index d1e9e982f..830e84ef2 100644 --- a/rcl/CMakeLists.txt +++ b/rcl/CMakeLists.txt @@ -34,6 +34,7 @@ set(${PROJECT_NAME}_sources src/rcl/graph.c src/rcl/guard_condition.c src/rcl/lexer.c + src/rcl/lexer_lookahead.c src/rcl/node.c src/rcl/publisher.c src/rcl/rcl.c diff --git a/rcl/include/rcl/lexer_lookahead.h b/rcl/include/rcl/lexer_lookahead.h new file mode 100644 index 000000000..7c9fc6d73 --- /dev/null +++ b/rcl/include/rcl/lexer_lookahead.h @@ -0,0 +1,240 @@ +// Copyright 2018 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RCL__LEXER_LOOKAHEAD_H_ +#define RCL__LEXER_LOOKAHEAD_H_ + +#include + +#include "rcl/allocator.h" +#include "rcl/lexer.h" +#include "rcl/macros.h" +#include "rcl/types.h" +#include "rcl/visibility_control.h" + +#if __cplusplus +extern "C" +{ +#endif + +// Forward declaration +struct rcl_lexer_lookahead2_impl_t; + +/// Track lexical analysis and allow looking ahead 2 lexemes. +typedef struct rcl_lexer_lookahead2_t +{ + struct rcl_lexer_lookahead2_impl_t * impl; +} rcl_lexer_lookahead2_t; + +/// Get a zero initialized rcl_lexer_lookahead2_t instance. +/** + * \sa rcl_lexer_lookahead2_init() + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | No + * Thread-Safe | Yes + * Uses Atomics | No + * Lock-Free | Yes + * + * \return zero initialized lookahead2 buffer. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_lexer_lookahead2_t +rcl_get_zero_initialized_lexer_lookahead2(); + +/// Initialize an rcl_lexer_lookahead2_t instance. +/** + * The lookahead2 buffer borrows a reference to the provided text. + * The text must not be freed before the buffer is finalized. + * The lookahead2 buffer only needs to be finalized if this function does not return RCL_RET_OK. + * \sa rcl_lexer_lookahead2_fini() + * + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | Yes + * Thread-Safe | No + * Uses Atomics | No + * Lock-Free | Yes + * + * \param[in] buffer A buffer that is zero initialized. + * \sa rcl_get_zero_initialized_lexer_lookahead2() + * \param[in] text The string to analyze. + * \param[in] allocator An allocator to use if an error occurs. + * \return `RCL_RET_OK` if the buffer is successfully initialized, or + * \return `RCL_RET_INVALID_ARGUMENT` if any function arguments are invalid, or + * \return `RCL_RET_BAD_ALLOC` if allocating memory failed, or + * \return `RCL_RET_ERROR` if an unspecified error occurrs. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_ret_t +rcl_lexer_lookahead2_init( + rcl_lexer_lookahead2_t * buffer, + const char * text, + rcl_allocator_t allocator); + +/// Finalize an instance of an rcl_lexer_lookahead2_t structure. +/** + * \sa rcl_lexer_lookahead2_init() + * + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | Yes [1] + * Thread-Safe | No + * Uses Atomics | No + * Lock-Free | Yes + * [1] Only allocates if an argument is invalid. + * + * \param[in] buffer The structure to be deallocated. + * \return `RCL_RET_OK` if the structure was successfully finalized, or + * \return `RCL_RET_INVALID_ARGUMENT` if any function arguments are invalid, or + * \return `RCL_RET_ERROR` if an unspecified error occurs. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_ret_t +rcl_lexer_lookahead2_fini( + rcl_lexer_lookahead2_t * buffer); + +/// Look ahead at the next lexeme in the string. +/** + * Repeated calls to peek will return the same lexeme. + * A parser that deems the next lexeme as valid must accept it to advance lexing. + * \sa rcl_lexer_lookahead2_accept() + * + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | Yes [1] + * Thread-Safe | No + * Uses Atomics | No + * Lock-Free | Yes + * [1] Only allocates if an argument is invalid or an internal bug is detected. + * + * \param[in] buffer the lookahead2 buffer being used to analyze a string. + * \param[out] next_type an output variable for the next lexeme in the string. + * \return `RCL_RET_OK` if peeking was successfull, or + * \return `RCL_RET_INVALID_ARGUMENT` if any function arguments are invalid, or + * \return `RCL_RET_ERROR` if an unspecified error occurs. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_ret_t +rcl_lexer_lookahead2_peek( + rcl_lexer_lookahead2_t * buffer, + rcl_lexeme_t * next_type); + +/// Look ahead at the next two lexemes in the string. +/** + * Repeated calls to peek2 will return the same two lexemes. + * A parser that deems the next two lexemes as valid must call accept twice to advance lexing. + * \sa rcl_lexer_lookahead2_accept() + * + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | Yes [1] + * Thread-Safe | No + * Uses Atomics | No + * Lock-Free | Yes + * [1] Only allocates if an argument is invalid or an internal bug is detected. + * + * \param[in] buffer the lookahead2 buffer being used to analyze a string. + * \param[out] next_type1 an output variable for the next lexeme in the string. + * \param[out] next_type2 an output variable for the lexeme after the next lexeme in the string. + * \return `RCL_RET_OK` if peeking was successfull, or + * \return `RCL_RET_INVALID_ARGUMENT` if any function arguments are invalid, or + * \return `RCL_RET_ERROR` if an unspecified error occurs. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_ret_t +rcl_lexer_lookahead2_peek2( + rcl_lexer_lookahead2_t * buffer, + rcl_lexeme_t * next_type1, + rcl_lexeme_t * next_type2); + +/// Accept a lexeme and advance analysis. +/** + * A token must have been peeked before it can be accepted. + * \sa rcl_lexer_lookahead2_peek() + * \sa rcl_lexer_lookahead2_peek2() + * + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | Yes [1] + * Thread-Safe | No + * Uses Atomics | No + * Lock-Free | Yes + * [1] Only allocates if an argument is invalid or an error occurs. + * + * \param[in] buffer the lookahead2 buffer being used to analyze a string. + * \param[out] lexeme_text pointer to where lexeme begins in string. + * \param[out] lexeme_text_length length of lexeme_text. + * \return `RCL_RET_OK` if peeking was successfull, or + * \return `RCL_RET_INVALID_ARGUMENT` if any function arguments are invalid, or + * \return `RCL_RET_ERROR` if an unspecified error occurs. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_ret_t +rcl_lexer_lookahead2_accept( + rcl_lexer_lookahead2_t * buffer, + const char ** lexeme_text, + size_t * lexeme_text_length); + +/// Require the next lexeme to be a certain type and advance analysis. +/** + * This method is a shortcut to peeking and accepting a lexeme. + * It should be used by a parser when there is only one valid lexeme that could come next. + * \sa rcl_lexer_lookahead2_peek() + * \sa rcl_lexer_lookahead2_accept() + * + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | Yes [1] + * Thread-Safe | No + * Uses Atomics | No + * Lock-Free | Yes + * [1] Only allocates if an argument is invalid or an error occurs. + * + * \param[in] buffer the lookahead2 buffer being used to analyze a string. + * \param[in] type the type the next lexeme must be. + * \param[out] lexeme_text pointer to where lexeme begins in string. + * \param[out] lexeme_text_length length of lexeme_text. + * \return `RCL_RET_OK` if the next lexeme was the expected one, or + * \return `RCL_RET_WRONG_LEXEME` if the next lexeme was not the expected one, or + * \return `RCL_RET_INVALID_ARGUMENT` if any function arguments are invalid, or + * \return `RCL_RET_ERROR` if an unspecified error occurs. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_ret_t +rcl_lexer_lookahead2_expect( + rcl_lexer_lookahead2_t * buffer, + rcl_lexeme_t type, + const char ** lexeme_text, + size_t * lexeme_text_length); + +#if __cplusplus +} +#endif + +#endif // RCL__LEXER_LOOKAHEAD_H_ diff --git a/rcl/include/rcl/types.h b/rcl/include/rcl/types.h index 93e9095d4..f63622a27 100644 --- a/rcl/include/rcl/types.h +++ b/rcl/include/rcl/types.h @@ -90,5 +90,7 @@ typedef rmw_ret_t rcl_ret_t; // rcl argument parsing specific ret codes in 1XXX /// Argument is not a valid remap rule #define RCL_RET_INVALID_REMAP_RULE 1001 +/// Expected one type of lexeme but got another +#define RCL_RET_WRONG_LEXEME 1002 #endif // RCL__TYPES_H_ diff --git a/rcl/src/rcl/lexer_lookahead.c b/rcl/src/rcl/lexer_lookahead.c new file mode 100644 index 000000000..f2a18412c --- /dev/null +++ b/rcl/src/rcl/lexer_lookahead.c @@ -0,0 +1,228 @@ +// Copyright 2018 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rcl/error_handling.h" +#include "rcl/lexer_lookahead.h" + +struct rcl_lexer_lookahead2_impl_t +{ + // Text that is being analyzed for lexemes + const char * text; + // Where in the text analysis is being performed + size_t text_idx; + + // index of first character of first terminal in buffer + size_t start1; + // One past last character of first terminal in buffer + size_t end1; + // Type of first terminal in buffer + rcl_lexeme_t type1; + + // index of first character of second terminal in buffer + size_t start2; + // One past last character of second terminal in buffer + size_t end2; + // Type of second terminal in buffer + rcl_lexeme_t type2; + + // Allocator to use if an error occurrs + rcl_allocator_t allocator; +}; + +rcl_lexer_lookahead2_t +rcl_get_zero_initialized_lexer_lookahead2() +{ + static rcl_lexer_lookahead2_t zero_initialized = { + .impl = NULL, + }; + return zero_initialized; +} + +rcl_ret_t +rcl_lexer_lookahead2_init( + rcl_lexer_lookahead2_t * buffer, + const char * text, + rcl_allocator_t allocator) +{ + RCL_CHECK_ALLOCATOR_WITH_MSG(&allocator, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); + RCL_CHECK_ARGUMENT_FOR_NULL(buffer, RCL_RET_INVALID_ARGUMENT, allocator); + RCL_CHECK_ARGUMENT_FOR_NULL(text, RCL_RET_INVALID_ARGUMENT, allocator); + if (NULL != buffer->impl) { + RCL_SET_ERROR_MSG("buffer must be zero initialized", allocator); + return RCL_RET_INVALID_ARGUMENT; + } + + buffer->impl = allocator.allocate(sizeof(struct rcl_lexer_lookahead2_impl_t), allocator.state); + RCL_CHECK_FOR_NULL_WITH_MSG( + buffer->impl, "Failed to allocate lookahead impl", return RCL_RET_BAD_ALLOC, allocator); + + buffer->impl->text = text; + buffer->impl->text_idx = 0; + buffer->impl->start1 = 0; + buffer->impl->end1 = 0; + buffer->impl->type1 = RCL_LEXEME_NONE; + buffer->impl->start2 = 0; + buffer->impl->end2 = 0; + buffer->impl->type2 = RCL_LEXEME_NONE; + buffer->impl->allocator = allocator; + + return RCL_RET_OK; +} + +rcl_ret_t +rcl_lexer_lookahead2_fini( + rcl_lexer_lookahead2_t * buffer) +{ + RCL_CHECK_ARGUMENT_FOR_NULL(buffer, RCL_RET_INVALID_ARGUMENT, rcl_get_default_allocator()); + RCL_CHECK_FOR_NULL_WITH_MSG( + buffer->impl, "buffer finalized twice", return RCL_RET_INVALID_ARGUMENT, + rcl_get_default_allocator()); + RCL_CHECK_ALLOCATOR_WITH_MSG( + &(buffer->impl->allocator), "invalid allocator", return RCL_RET_INVALID_ARGUMENT); + + buffer->impl->allocator.deallocate(buffer->impl, buffer->impl->allocator.state); + buffer->impl = NULL; + return RCL_RET_OK; +} + +rcl_ret_t +rcl_lexer_lookahead2_peek( + rcl_lexer_lookahead2_t * buffer, + rcl_lexeme_t * next_type) +{ + RCL_CHECK_ARGUMENT_FOR_NULL(buffer, RCL_RET_INVALID_ARGUMENT, rcl_get_default_allocator()); + RCL_CHECK_FOR_NULL_WITH_MSG( + buffer->impl, "buffer not initialized", return RCL_RET_INVALID_ARGUMENT, + rcl_get_default_allocator()); + RCL_CHECK_ARGUMENT_FOR_NULL(next_type, RCL_RET_INVALID_ARGUMENT, buffer->impl->allocator); + + rcl_ret_t ret; + size_t length; + + if (buffer->impl->text_idx >= buffer->impl->end1) { + // No buffered lexeme; get one + ret = rcl_lexer_analyze( + &(buffer->impl->text[buffer->impl->text_idx]), + buffer->impl->allocator, + &(buffer->impl->type1), + &length); + + if (RCL_RET_OK != ret) { + return ret; + } + + buffer->impl->start1 = buffer->impl->text_idx; + buffer->impl->end1 = buffer->impl->start1 + length; + } + + *next_type = buffer->impl->type1; + return RCL_RET_OK; +} + +rcl_ret_t +rcl_lexer_lookahead2_peek2( + rcl_lexer_lookahead2_t * buffer, + rcl_lexeme_t * next_type1, + rcl_lexeme_t * next_type2) +{ + rcl_ret_t ret; + // Peek 1 ahead first (reusing its error checking for buffer and next_type1) + ret = rcl_lexer_lookahead2_peek(buffer, next_type1); + if (RCL_RET_OK != ret) { + return ret; + } + RCL_CHECK_ARGUMENT_FOR_NULL(next_type2, RCL_RET_INVALID_ARGUMENT, buffer->impl->allocator); + + size_t length; + + if (buffer->impl->text_idx >= buffer->impl->end2) { + // No buffered lexeme; get one + ret = rcl_lexer_analyze( + &(buffer->impl->text[buffer->impl->end1]), + buffer->impl->allocator, + &(buffer->impl->type2), + &length); + + if (RCL_RET_OK != ret) { + return ret; + } + + buffer->impl->start2 = buffer->impl->end1; + buffer->impl->end2 = buffer->impl->start2 + length; + } + + *next_type2 = buffer->impl->type2; + return RCL_RET_OK; +} + +rcl_ret_t +rcl_lexer_lookahead2_accept( + rcl_lexer_lookahead2_t * buffer, + const char ** lexeme_text, + size_t * lexeme_text_length) +{ + RCL_CHECK_ARGUMENT_FOR_NULL(buffer, RCL_RET_INVALID_ARGUMENT, rcl_get_default_allocator()); + RCL_CHECK_FOR_NULL_WITH_MSG( + buffer->impl, "buffer not initialized", return RCL_RET_INVALID_ARGUMENT, + rcl_get_default_allocator()); + RCL_CHECK_ARGUMENT_FOR_NULL(lexeme_text, RCL_RET_INVALID_ARGUMENT, buffer->impl->allocator); + RCL_CHECK_ARGUMENT_FOR_NULL( + lexeme_text_length, RCL_RET_INVALID_ARGUMENT, buffer->impl->allocator); + + if (RCL_LEXEME_EOF == buffer->impl->type1) { + // Reached EOF, nothing to accept + *lexeme_text = &(buffer->impl->text[buffer->impl->text_idx]); + *lexeme_text_length = 0; + return RCL_RET_OK; + } + + if (buffer->impl->text_idx >= buffer->impl->end1) { + RCL_SET_ERROR_MSG("no lexeme to accept", buffer->impl->allocator); + return RCL_RET_ERROR; + } + + *lexeme_text = &(buffer->impl->text[buffer->impl->start1]); + *lexeme_text_length = buffer->impl->end1 - buffer->impl->start1; + + // Advance lexer position + buffer->impl->text_idx = buffer->impl->end1; + + // Move second lexeme in buffer to first position + buffer->impl->start1 = buffer->impl->start2; + buffer->impl->end1 = buffer->impl->end2; + buffer->impl->type1 = buffer->impl->type2; + + return RCL_RET_OK; +} + +rcl_ret_t +rcl_lexer_lookahead2_expect( + rcl_lexer_lookahead2_t * buffer, + rcl_lexeme_t type, + const char ** lexeme_text, + size_t * lexeme_text_length) +{ + rcl_ret_t ret; + rcl_lexeme_t lexeme; + + ret = rcl_lexer_lookahead2_peek(buffer, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + if (type != lexeme) { + RCL_SET_ERROR_MSG("Unexpected lexeme", buffer->impl->allocator); + return RCL_RET_WRONG_LEXEME; + } + return rcl_lexer_lookahead2_accept(buffer, lexeme_text, lexeme_text_length); +} diff --git a/rcl/test/CMakeLists.txt b/rcl/test/CMakeLists.txt index 4ae5ed8ea..6c8e6ad63 100644 --- a/rcl/test/CMakeLists.txt +++ b/rcl/test/CMakeLists.txt @@ -75,6 +75,14 @@ function(test_target_function) AMENT_DEPENDENCIES ${rmw_implementation} ) + rcl_add_custom_gtest(test_lexer_lookahead${target_suffix} + SRCS rcl/test_lexer_lookahead.cpp + ENV ${extra_test_env} + APPEND_LIBRARY_DIRS ${extra_lib_dirs} + LIBRARIES ${PROJECT_NAME} ${extra_test_libraries} + AMENT_DEPENDENCIES ${rmw_implementation} + ) + set(SKIP_TEST "") # TODO(wjwwood): remove this when the graph API works properly for connext dynamic if( diff --git a/rcl/test/rcl/test_lexer_lookahead.cpp b/rcl/test/rcl/test_lexer_lookahead.cpp new file mode 100644 index 000000000..694d7bd1e --- /dev/null +++ b/rcl/test/rcl/test_lexer_lookahead.cpp @@ -0,0 +1,286 @@ +// Copyright 2018 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include "../scope_exit.hpp" + +#include "rcl/error_handling.h" +#include "rcl/lexer_lookahead.h" + +#ifdef RMW_IMPLEMENTATION +# define CLASSNAME_(NAME, SUFFIX) NAME ## __ ## SUFFIX +# define CLASSNAME(NAME, SUFFIX) CLASSNAME_(NAME, SUFFIX) +#else +# define CLASSNAME(NAME, SUFFIX) NAME +#endif + +class CLASSNAME (TestLexerLookaheadFixture, RMW_IMPLEMENTATION) : public ::testing::Test +{ +public: + void SetUp() + { + } + + void TearDown() + { + } +}; + +#define SCOPE_LOOKAHEAD2(name, text) \ + { \ + name = rcl_get_zero_initialized_lexer_lookahead2(); \ + rcl_ret_t ret = rcl_lexer_lookahead2_init(&name, text, rcl_get_default_allocator()); \ + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); \ + } \ + auto __scope_lookahead2_ ## name = make_scope_exit( \ + [&name]() { \ + rcl_ret_t ret = rcl_lexer_lookahead2_fini(&buffer); \ + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); \ + }) + +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_init_fini_twice) +{ + rcl_lexer_lookahead2_t buffer = rcl_get_zero_initialized_lexer_lookahead2(); + rcl_ret_t ret = rcl_lexer_lookahead2_init(&buffer, "foobar", rcl_get_default_allocator()); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + + ret = rcl_lexer_lookahead2_fini(&buffer); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + + ret = rcl_lexer_lookahead2_fini(&buffer); + EXPECT_EQ(RCL_RET_INVALID_ARGUMENT, ret); + rcl_reset_error(); +} + +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_peek) +{ + rcl_ret_t ret; + rcl_lexer_lookahead2_t buffer; + SCOPE_LOOKAHEAD2(buffer, "foobar"); + + rcl_lexeme_t lexeme = RCL_LEXEME_NONE; + + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret); + EXPECT_EQ(RCL_LEXEME_TOKEN, lexeme); + + // Test again to make sure peek isn't advancing the lexer + lexeme = RCL_LEXEME_NONE; + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret); + EXPECT_EQ(RCL_LEXEME_TOKEN, lexeme); +} + +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_peek2) +{ + rcl_ret_t ret; + rcl_lexer_lookahead2_t buffer; + SCOPE_LOOKAHEAD2(buffer, "foobar/"); + + rcl_lexeme_t lexeme1 = RCL_LEXEME_NONE; + rcl_lexeme_t lexeme2 = RCL_LEXEME_NONE; + + ret = rcl_lexer_lookahead2_peek2(&buffer, &lexeme1, &lexeme2); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_EQ(RCL_LEXEME_TOKEN, lexeme1); + EXPECT_EQ(RCL_LEXEME_FORWARD_SLASH, lexeme2); + + // Test again to make sure peek2 isn't advancing the lexer + lexeme1 = RCL_LEXEME_NONE; + lexeme2 = RCL_LEXEME_NONE; + ret = rcl_lexer_lookahead2_peek2(&buffer, &lexeme1, &lexeme2); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_EQ(RCL_LEXEME_TOKEN, lexeme1); + EXPECT_EQ(RCL_LEXEME_FORWARD_SLASH, lexeme2); +} + +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_accept) +{ + rcl_ret_t ret; + rcl_lexer_lookahead2_t buffer; + SCOPE_LOOKAHEAD2(buffer, "foobar/"); + + rcl_lexeme_t lexeme = RCL_LEXEME_NONE; + const char * lexeme_text; + size_t lexeme_text_length; + + // Peek token + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_EQ(RCL_LEXEME_TOKEN, lexeme); + + // accept token + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("foobar", std::string(lexeme_text, lexeme_text_length).c_str()); + + // peek forward slash + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_EQ(RCL_LEXEME_FORWARD_SLASH, lexeme); + + // accept forward slash + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("/", std::string(lexeme_text, lexeme_text_length).c_str()); + + // peek eof + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_EQ(RCL_LEXEME_EOF, lexeme); + + // accept eof + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("", std::string(lexeme_text, lexeme_text_length).c_str()); + + // peek eof again + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_EQ(RCL_LEXEME_EOF, lexeme); +} + +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_expect) +{ + rcl_ret_t ret; + rcl_lexer_lookahead2_t buffer; + SCOPE_LOOKAHEAD2(buffer, "node_name:__node:=new_1"); + const char * lexeme_text; + size_t lexeme_text_length; + + ret = rcl_lexer_lookahead2_expect(&buffer, RCL_LEXEME_TOKEN, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("node_name", std::string(lexeme_text, lexeme_text_length).c_str()); + + ret = rcl_lexer_lookahead2_expect( + &buffer, RCL_LEXEME_FORWARD_SLASH, &lexeme_text, &lexeme_text_length); + EXPECT_EQ(RCL_RET_WRONG_LEXEME, ret) << rcl_get_error_string_safe(); +} + +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_lex_long_string) +{ + rcl_ret_t ret; + rcl_lexer_lookahead2_t buffer; + SCOPE_LOOKAHEAD2(buffer, ":\\1rostopic://\\2rosservice://~/\\8:=**:*foobar"); + const char * lexeme_text; + size_t lexeme_text_length; + rcl_lexeme_t lexeme; + + // : + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_COLON, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ(":", std::string(lexeme_text, lexeme_text_length).c_str()); + + // \1 + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_BR1, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("\\1", std::string(lexeme_text, lexeme_text_length).c_str()); + + // rostopic:// + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_URL_TOPIC, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("rostopic://", std::string(lexeme_text, lexeme_text_length).c_str()); + + // \2 + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_BR2, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("\\2", std::string(lexeme_text, lexeme_text_length).c_str()); + + // rosservice:// + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_URL_SERVICE, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("rosservice://", std::string(lexeme_text, lexeme_text_length).c_str()); + + // ~/ + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_TILDE_SLASH, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("~/", std::string(lexeme_text, lexeme_text_length).c_str()); + + // \8 + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_BR8, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("\\8", std::string(lexeme_text, lexeme_text_length).c_str()); + + // := + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_SEPARATOR, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ(":=", std::string(lexeme_text, lexeme_text_length).c_str()); + + // ** + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_WILD_MULTI, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("**", std::string(lexeme_text, lexeme_text_length).c_str()); + + // : + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_COLON, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ(":", std::string(lexeme_text, lexeme_text_length).c_str()); + + // * + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_WILD_ONE, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("*", std::string(lexeme_text, lexeme_text_length).c_str()); + + // foobar + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_TOKEN, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("foobar", std::string(lexeme_text, lexeme_text_length).c_str()); + + // eof + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_LEXEME_EOF, lexeme); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_STREQ("", std::string(lexeme_text, lexeme_text_length).c_str()); +} From 6a82306adc92fd5e0741da5ebe0049d8a66d0f2a Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 18:00:21 -0700 Subject: [PATCH 13/38] Allow ignoring lexeme text --- rcl/src/rcl/lexer_lookahead.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rcl/src/rcl/lexer_lookahead.c b/rcl/src/rcl/lexer_lookahead.c index f2a18412c..51f4ce4d1 100644 --- a/rcl/src/rcl/lexer_lookahead.c +++ b/rcl/src/rcl/lexer_lookahead.c @@ -176,9 +176,9 @@ rcl_lexer_lookahead2_accept( RCL_CHECK_FOR_NULL_WITH_MSG( buffer->impl, "buffer not initialized", return RCL_RET_INVALID_ARGUMENT, rcl_get_default_allocator()); - RCL_CHECK_ARGUMENT_FOR_NULL(lexeme_text, RCL_RET_INVALID_ARGUMENT, buffer->impl->allocator); - RCL_CHECK_ARGUMENT_FOR_NULL( - lexeme_text_length, RCL_RET_INVALID_ARGUMENT, buffer->impl->allocator); + if ((NULL == lexeme_text || NULL == lexeme_text_length) && lexeme_text != lexeme_text_length) { + RCL_SET_ERROR_MSG("text and length must both be set or both be NULL", buffer->impl->allocator); + } if (RCL_LEXEME_EOF == buffer->impl->type1) { // Reached EOF, nothing to accept From 4423f030f63d78efd40de1ab56ee66cf752bb63d Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 6 Apr 2018 18:00:51 -0700 Subject: [PATCH 14/38] Beginnings of recursive descent parser --- rcl/src/rcl/arguments.c | 231 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/rcl/src/rcl/arguments.c b/rcl/src/rcl/arguments.c index b40ee46bb..3b28c6ceb 100644 --- a/rcl/src/rcl/arguments.c +++ b/rcl/src/rcl/arguments.c @@ -19,6 +19,7 @@ #include "./arguments_impl.h" #include "./remap_impl.h" #include "rcl/error_handling.h" +#include "rcl/lexer_lookahead.h" #include "rcl/validate_topic_name.h" #include "rcutils/allocator.h" #include "rcutils/logging_macros.h" @@ -34,6 +35,236 @@ extern "C" // Instance of global arguments. static rcl_arguments_t __rcl_global_arguments; +/// Parses a fully qualified namespace for a namespace replacement rule (ex: `/foo/bar`) +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_fully_qualified_namespace( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + return RCL_RET_OK; +} + +/// Parse either a token or a backreference (ex: `bar`, or `\7`). +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_replacement_token( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + return RCL_RET_OK; +} + +/// Parse the replacement side of a name remapping rule (ex: `bar/\1/foo`). +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_replacement_name( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + return RCL_RET_OK; +} + +/// Parse either a token or a wildcard (ex: `foobar`, or `*`, or `**`). +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_match_token( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + return RCL_RET_OK; +} + +/// Parse the match side of a name remapping rule (ex: `rostopic://foo`) +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_match_name( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + return RCL_RET_OK; +} + +/// Parse a name remapping rule (ex: `rostopic:///foo:=bar`). +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_name_remap( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + return RCL_RET_OK; +} + +/// Parse a namespace replacement rule (ex: `__ns:=/new/ns`). +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_namespace_replacement( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + rcl_ret_t ret; + // __ns + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_NS, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + // := + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_SEPARATOR, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + // /foo/bar + // TODO(sloretz) rcl_lexer_lookahead2_current_text(lex_lookahead) to return the current text ptr + return _rcl_parse_remap_fully_qualified_namespace(lex_lookahead, rule); +} + +/// Parse a nodename replacement rule (ex: `__node:=new_name`). +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_nodename_replacement( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + rcl_ret_t ret; + const char * node_name; + size_t length; + + // __node + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_NODE, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + // := + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_SEPARATOR, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + // new_node_name + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_TOKEN, &node_name, &length); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + + // copy the node name into the replacement side of the rule + rule->replacement = rcutils_strndup(node_name, length, rule->allocator); + if (NULL == rule->replacement) { + RCL_SET_ERROR_MSG("failed to allocate node name", rule->allocator); + ret = RCL_RET_BAD_ALLOC; + } + + return ret; +} + +/// Parse a nodename prefix including trailing colon (ex: `node_name:`). +/// \sa _rcl_parse_remap_begin_remap_rule() +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_nodename_prefix( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + rcl_ret_t ret; + const char * node_name; + size_t length; + + // Expect a token and a colon + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_TOKEN, &node_name, &length); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_COLON, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + + // copy the node name into the rule + rule->node_name = rcutils_strndup(node_name, length, rule->allocator); + if (NULL == rule->node_name) { + RCL_SET_ERROR_MSG("failed to allocate node name", rule->allocator); + ret = RCL_RET_BAD_ALLOC; + } + + return ret; +} + +/// Start recursive descent parseing of a remap rule. +/// \param[in] lex_lookahead a lookahead(2) buffer for the parser to use. +/// \param[in,out] rule input a zero intialized rule, output a fully initialized one. +/// \return RCL_RET_OK if a valid rule was parsed, or +/// \return RCL_RET_INVALID_REMAP_RULE if the argument is not a valid rule, or +/// \return RCL_RET_BAD_ALLOC if an allocation failed, or +/// \return RLC_RET_ERROR if an unspecified error occurred. +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_begin_remap_rule( + rcl_lexer_lookahead2_t * lex_lookahead, + rcl_remap_t * rule) +{ + rcl_ret_t ret; + rcl_lexeme_t lexeme1; + rcl_lexeme_t lexeme2; + + // Check for optional nodename prefix + ret = rcl_lexer_lookahead2_peek2(lex_lookahead, &lexeme1, &lexeme2); + if (RCL_RET_OK != ret) { + return ret; + } + if (RCL_LEXEME_TOKEN == lexeme1 && RCL_LEXEME_COLON == lexeme2) { + ret = _rcl_parse_remap_nodename_prefix(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + } + + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme1); + if (RCL_RET_OK != ret) { + return ret; + } + + // What type of rule is this (node name replacement, namespace replacement, or name remap)? + if (RCL_LEXEME_NODE == lexeme1) { + ret = _rcl_parse_remap_nodename_replacement(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + } else if (RCL_LEXEME_NS == lexeme1) { + ret = _rcl_parse_remap_namespace_replacement(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + } else { + ret = _rcl_parse_remap_name_remap(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + } + + // Make sure all characters in string have been consumed + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_EOF, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + return ret; +} + /// Parse an argument that may or may not be a remap rule. /// \param[in] arg the argument to parse /// \param[in] allocator an allocator to use From 884e286f0f07f35f43ce3143ffe5918a6c6d297d Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 09:28:40 -0700 Subject: [PATCH 15/38] Add method to get current position in text --- rcl/include/rcl/lexer_lookahead.h | 21 +++++++++++++++++++++ rcl/src/rcl/lexer_lookahead.c | 29 ++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/rcl/include/rcl/lexer_lookahead.h b/rcl/include/rcl/lexer_lookahead.h index 7c9fc6d73..14c5b34a9 100644 --- a/rcl/include/rcl/lexer_lookahead.h +++ b/rcl/include/rcl/lexer_lookahead.h @@ -233,6 +233,27 @@ rcl_lexer_lookahead2_expect( const char ** lexeme_text, size_t * lexeme_text_length); +/// Get the text at the point where it is currently being analyzed. +/** + *
+ * Attribute | Adherence + * ------------------ | ------------- + * Allocates Memory | No + * Thread-Safe | Yes + * Uses Atomics | No + * Lock-Free | Yes + * + * \param[in] buffer the lookahead2 buffer being used to analyze a string. + * \return a pointer inside the original text at the position being analyzed, or + * \return `NULL` if buffer is itself `NULL` or zero initialized, or + * \return an undefined value if buffer is not initialized or has been finalized. + */ +RCL_PUBLIC +RCL_WARN_UNUSED +const char * +rcl_lexer_lookahead2_get_text( + const rcl_lexer_lookahead2_t * buffer); + #if __cplusplus } #endif diff --git a/rcl/src/rcl/lexer_lookahead.c b/rcl/src/rcl/lexer_lookahead.c index 51f4ce4d1..34f849d4a 100644 --- a/rcl/src/rcl/lexer_lookahead.c +++ b/rcl/src/rcl/lexer_lookahead.c @@ -113,7 +113,7 @@ rcl_lexer_lookahead2_peek( if (buffer->impl->text_idx >= buffer->impl->end1) { // No buffered lexeme; get one ret = rcl_lexer_analyze( - &(buffer->impl->text[buffer->impl->text_idx]), + rcl_lexer_lookahead2_get_text(buffer), buffer->impl->allocator, &(buffer->impl->type1), &length); @@ -176,14 +176,19 @@ rcl_lexer_lookahead2_accept( RCL_CHECK_FOR_NULL_WITH_MSG( buffer->impl, "buffer not initialized", return RCL_RET_INVALID_ARGUMENT, rcl_get_default_allocator()); - if ((NULL == lexeme_text || NULL == lexeme_text_length) && lexeme_text != lexeme_text_length) { + if ( + (NULL == lexeme_text && NULL != lexeme_text_length) || + (NULL != lexeme_text && NULL == lexeme_text_length)) + { RCL_SET_ERROR_MSG("text and length must both be set or both be NULL", buffer->impl->allocator); } if (RCL_LEXEME_EOF == buffer->impl->type1) { // Reached EOF, nothing to accept - *lexeme_text = &(buffer->impl->text[buffer->impl->text_idx]); - *lexeme_text_length = 0; + if (NULL != lexeme_text && NULL != lexeme_text_length) { + *lexeme_text = rcl_lexer_lookahead2_get_text(buffer); + *lexeme_text_length = 0; + } return RCL_RET_OK; } @@ -192,8 +197,10 @@ rcl_lexer_lookahead2_accept( return RCL_RET_ERROR; } - *lexeme_text = &(buffer->impl->text[buffer->impl->start1]); - *lexeme_text_length = buffer->impl->end1 - buffer->impl->start1; + if (NULL != lexeme_text && NULL != lexeme_text_length) { + *lexeme_text = &(buffer->impl->text[buffer->impl->start1]); + *lexeme_text_length = buffer->impl->end1 - buffer->impl->start1; + } // Advance lexer position buffer->impl->text_idx = buffer->impl->end1; @@ -221,8 +228,16 @@ rcl_lexer_lookahead2_expect( return ret; } if (type != lexeme) { - RCL_SET_ERROR_MSG("Unexpected lexeme", buffer->impl->allocator); + RCL_SET_ERROR_MSG_WITH_FORMAT_STRING( + buffer->impl->allocator, "Expected %d got %d at %lu", type, lexeme, buffer->impl->text_idx); return RCL_RET_WRONG_LEXEME; } return rcl_lexer_lookahead2_accept(buffer, lexeme_text, lexeme_text_length); } + +const char * +rcl_lexer_lookahead2_get_text( + const rcl_lexer_lookahead2_t * buffer) +{ + return &(buffer->impl->text[buffer->impl->text_idx]); +} From e60841f4bf4b8b2fe19a2458cae83617039208d9 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 09:29:03 -0700 Subject: [PATCH 16/38] new remap parsing passes unit tests --- rcl/src/rcl/arguments.c | 414 ++++++++++++++++++++++++---------------- 1 file changed, 253 insertions(+), 161 deletions(-) diff --git a/rcl/src/rcl/arguments.c b/rcl/src/rcl/arguments.c index 3b28c6ceb..90ce4b990 100644 --- a/rcl/src/rcl/arguments.c +++ b/rcl/src/rcl/arguments.c @@ -41,9 +41,29 @@ static rcl_arguments_t __rcl_global_arguments; RCL_LOCAL rcl_ret_t _rcl_parse_remap_fully_qualified_namespace( - rcl_lexer_lookahead2_t * lex_lookahead, - rcl_remap_t * rule) + rcl_lexer_lookahead2_t * lex_lookahead) { + rcl_ret_t ret; + + // Must have at least one Forward slash / + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_FORWARD_SLASH, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + + // repeated tokens and slashes (allow trailing slash, but don't require it) + while (true) { + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_TOKEN, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + rcl_reset_error(); + break; + } + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_FORWARD_SLASH, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + rcl_reset_error(); + break; + } + } return RCL_RET_OK; } @@ -56,7 +76,28 @@ _rcl_parse_remap_replacement_token( rcl_lexer_lookahead2_t * lex_lookahead, rcl_remap_t * rule) { - return RCL_RET_OK; + rcl_ret_t ret; + rcl_lexeme_t lexeme; + + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + + if ( + RCL_LEXEME_BR1 == lexeme || RCL_LEXEME_BR2 == lexeme || RCL_LEXEME_BR3 == lexeme || + RCL_LEXEME_BR4 == lexeme || RCL_LEXEME_BR5 == lexeme || RCL_LEXEME_BR6 == lexeme || + RCL_LEXEME_BR7 == lexeme || RCL_LEXEME_BR8 == lexeme || RCL_LEXEME_BR9 == lexeme) + { + RCL_SET_ERROR_MSG("Backreferences are not implemented", rule->allocator); + return RCL_RET_ERROR; + } else if (RCL_LEXEME_TOKEN == lexeme) { + ret = rcl_lexer_lookahead2_accept(lex_lookahead, NULL, NULL); + } else { + ret = RCL_RET_INVALID_REMAP_RULE; + } + + return ret; } /// Parse the replacement side of a name remapping rule (ex: `bar/\1/foo`). @@ -68,6 +109,60 @@ _rcl_parse_remap_replacement_name( rcl_lexer_lookahead2_t * lex_lookahead, rcl_remap_t * rule) { + rcl_ret_t ret; + rcl_lexeme_t lexeme; + + const char * replacement_start = rcl_lexer_lookahead2_get_text(lex_lookahead); + if (NULL == replacement_start) { + RCL_SET_ERROR_MSG("failed to get start of replacement", rule->allocator); + return RCL_RET_ERROR; + } + + // private name (~/...) or fully qualified name (/...) ? + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + if (RCL_LEXEME_TILDE_SLASH == lexeme || RCL_LEXEME_FORWARD_SLASH == lexeme) { + ret = rcl_lexer_lookahead2_accept(lex_lookahead, NULL, NULL); + } + if (RCL_RET_OK != ret) { + return ret; + } + + // token ( '/' token )* + ret = _rcl_parse_remap_replacement_token(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + while (RCL_LEXEME_EOF != lexeme) { + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_FORWARD_SLASH, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + ret = _rcl_parse_remap_replacement_token(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + } + + // Copy replacement into rule + const char * replacement_end = rcl_lexer_lookahead2_get_text(lex_lookahead); + size_t length = (size_t)(replacement_end - replacement_start); + rule->replacement = rcutils_strndup(replacement_start, length, rule->allocator); + if (NULL == rule->replacement) { + RCL_SET_ERROR_MSG("failed to copy replacement", rule->allocator); + return RCL_RET_BAD_ALLOC; + } + return RCL_RET_OK; } @@ -80,7 +175,27 @@ _rcl_parse_remap_match_token( rcl_lexer_lookahead2_t * lex_lookahead, rcl_remap_t * rule) { - return RCL_RET_OK; + rcl_ret_t ret; + rcl_lexeme_t lexeme; + + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + + if (RCL_LEXEME_TOKEN == lexeme) { + ret = rcl_lexer_lookahead2_accept(lex_lookahead, NULL, NULL); + } else if (RCL_LEXEME_WILD_ONE == lexeme) { + RCL_SET_ERROR_MSG("Wildcard '*' is not implemented", rule->allocator); + return RCL_RET_ERROR; + } else if (RCL_LEXEME_WILD_MULTI == lexeme) { + RCL_SET_ERROR_MSG("Wildcard '**' is not implemented", rule->allocator); + return RCL_RET_ERROR; + } else { + ret = RCL_RET_INVALID_REMAP_RULE; + } + + return ret; } /// Parse the match side of a name remapping rule (ex: `rostopic://foo`) @@ -92,6 +207,77 @@ _rcl_parse_remap_match_name( rcl_lexer_lookahead2_t * lex_lookahead, rcl_remap_t * rule) { + rcl_ret_t ret; + rcl_lexeme_t lexeme; + // rostopic:// rosservice:// + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + if (RCL_LEXEME_URL_SERVICE == lexeme) { + rule->type = RCL_SERVICE_REMAP; + ret = rcl_lexer_lookahead2_accept(lex_lookahead, NULL, NULL); + } else if (RCL_LEXEME_URL_TOPIC == lexeme) { + rule->type = RCL_TOPIC_REMAP; + ret = rcl_lexer_lookahead2_accept(lex_lookahead, NULL, NULL); + } else { + rule->type = (RCL_TOPIC_REMAP | RCL_SERVICE_REMAP); + } + if (RCL_RET_OK != ret) { + return ret; + } + + const char * match_start = rcl_lexer_lookahead2_get_text(lex_lookahead); + if (NULL == match_start) { + RCL_SET_ERROR_MSG("failed to get start of match", rule->allocator); + return RCL_RET_ERROR; + } + + // private name (~/...) or fully qualified name (/...) ? + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + if (RCL_LEXEME_TILDE_SLASH == lexeme || RCL_LEXEME_FORWARD_SLASH == lexeme) { + ret = rcl_lexer_lookahead2_accept(lex_lookahead, NULL, NULL); + } + if (RCL_RET_OK != ret) { + return ret; + } + + // token ( '/' token )* + ret = _rcl_parse_remap_match_token(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + while (RCL_LEXEME_SEPARATOR != lexeme) { + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_FORWARD_SLASH, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + ret = _rcl_parse_remap_match_token(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + ret = rcl_lexer_lookahead2_peek(lex_lookahead, &lexeme); + if (RCL_RET_OK != ret) { + return ret; + } + } + + // Copy match into rule + const char * match_end = rcl_lexer_lookahead2_get_text(lex_lookahead); + size_t length = (size_t)(match_end - match_start); + rule->match = rcutils_strndup(match_start, length, rule->allocator); + if (NULL == rule->match) { + RCL_SET_ERROR_MSG("failed to copy match", rule->allocator); + return RCL_RET_BAD_ALLOC; + } + return RCL_RET_OK; } @@ -104,6 +290,23 @@ _rcl_parse_remap_name_remap( rcl_lexer_lookahead2_t * lex_lookahead, rcl_remap_t * rule) { + rcl_ret_t ret; + // match + ret = _rcl_parse_remap_match_name(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + // := + ret = rcl_lexer_lookahead2_expect(lex_lookahead, RCL_LEXEME_SEPARATOR, NULL, NULL); + if (RCL_RET_WRONG_LEXEME == ret) { + return RCL_RET_INVALID_REMAP_RULE; + } + // replacement + ret = _rcl_parse_remap_replacement_name(lex_lookahead, rule); + if (RCL_RET_OK != ret) { + return ret; + } + return RCL_RET_OK; } @@ -128,8 +331,27 @@ _rcl_parse_remap_namespace_replacement( return RCL_RET_INVALID_REMAP_RULE; } // /foo/bar - // TODO(sloretz) rcl_lexer_lookahead2_current_text(lex_lookahead) to return the current text ptr - return _rcl_parse_remap_fully_qualified_namespace(lex_lookahead, rule); + const char * ns_start = rcl_lexer_lookahead2_get_text(lex_lookahead); + if (NULL == ns_start) { + RCL_SET_ERROR_MSG("failed to get start of namespace", rule->allocator); + return RCL_RET_ERROR; + } + ret = _rcl_parse_remap_fully_qualified_namespace(lex_lookahead); + if (RCL_RET_OK != ret) { + return ret; + } + + // Copy namespace into rule + const char * ns_end = rcl_lexer_lookahead2_get_text(lex_lookahead); + size_t length = (size_t)(ns_end - ns_start); + rule->replacement = rcutils_strndup(ns_start, length, rule->allocator); + if (NULL == rule->replacement) { + RCL_SET_ERROR_MSG("failed to copy namespace", rule->allocator); + return RCL_RET_BAD_ALLOC; + } + + rule->type = RCL_NAMESPACE_REMAP; + return RCL_RET_OK; } /// Parse a nodename replacement rule (ex: `__node:=new_name`). @@ -160,15 +382,18 @@ _rcl_parse_remap_nodename_replacement( if (RCL_RET_WRONG_LEXEME == ret) { return RCL_RET_INVALID_REMAP_RULE; } - + if (RCL_RET_OK != ret) { + return ret; + } // copy the node name into the replacement side of the rule rule->replacement = rcutils_strndup(node_name, length, rule->allocator); if (NULL == rule->replacement) { RCL_SET_ERROR_MSG("failed to allocate node name", rule->allocator); - ret = RCL_RET_BAD_ALLOC; + return RCL_RET_BAD_ALLOC; } - return ret; + rule->type = RCL_NODENAME_REMAP; + return RCL_RET_OK; } /// Parse a nodename prefix including trailing colon (ex: `node_name:`). @@ -198,13 +423,13 @@ _rcl_parse_remap_nodename_prefix( rule->node_name = rcutils_strndup(node_name, length, rule->allocator); if (NULL == rule->node_name) { RCL_SET_ERROR_MSG("failed to allocate node name", rule->allocator); - ret = RCL_RET_BAD_ALLOC; + return RCL_RET_BAD_ALLOC; } - return ret; + return RCL_RET_OK; } -/// Start recursive descent parseing of a remap rule. +/// Start recursive descent parsing of a remap rule. /// \param[in] lex_lookahead a lookahead(2) buffer for the parser to use. /// \param[in,out] rule input a zero intialized rule, output a fully initialized one. /// \return RCL_RET_OK if a valid rule was parsed, or @@ -284,163 +509,30 @@ _rcl_parse_remap_rule( RCL_CHECK_ARGUMENT_FOR_NULL(arg, RCL_RET_INVALID_ARGUMENT, allocator); RCL_CHECK_ARGUMENT_FOR_NULL(output_rule, RCL_RET_INVALID_ARGUMENT, allocator); - size_t len_node_name = 0; - size_t len_match = 0; - size_t len_replacement = 0; - - const char * separator = NULL; - const char * colon = NULL; - const char * match_begin = arg; - const char * replacement_begin = NULL; - - // A valid rule has two parts separated by := - separator = strstr(arg, ":="); - if (NULL == separator) { - RCL_SET_ERROR_MSG("missing :=", allocator); - return RCL_RET_INVALID_REMAP_RULE; - } - - replacement_begin = separator + 2; - - // must have characters on both sides of the separator - len_match = separator - arg; - len_replacement = strlen(replacement_begin); - if (0 == len_match) { - RCL_SET_ERROR_MSG("match is zero length", allocator); - return RCL_RET_INVALID_REMAP_RULE; - } else if (0 == len_replacement) { - RCL_SET_ERROR_MSG("replacement has zero length", allocator); - return RCL_RET_INVALID_REMAP_RULE; - } - - colon = strchr(arg, ':'); - if (NULL != colon) { - if (colon < separator) { - // If there is a : on the match side then there is a node-name prefix - match_begin = colon + 1; - len_node_name = colon - arg; - len_match = separator - match_begin; - // node name must have at least one character - if (len_node_name <= 0) { - RCL_SET_ERROR_MSG("node name previx has zero length", allocator); - return RCL_RET_INVALID_REMAP_RULE; - } - } else if (colon > separator) { - // If the colon is on the replacement side then this couldn't be a valid rule - RCL_SET_ERROR_MSG("replacement side cannot contain a :", allocator); - return RCL_RET_INVALID_REMAP_RULE; - } - } + rcl_ret_t ret; - // Maybe match length changed because there was a node name prefix - if (0 == len_match) { - RCL_SET_ERROR_MSG("match is zero length", allocator); - return RCL_RET_INVALID_REMAP_RULE; - } + output_rule->allocator = allocator; + rcl_lexer_lookahead2_t lex_lookahead = rcl_get_zero_initialized_lexer_lookahead2(); - // Make sure node name contains only valid characters - if (len_node_name) { - int validation_result; - size_t invalid_index; - rmw_ret_t rmw_ret = rmw_validate_node_name_with_size( - arg, len_node_name, &validation_result, &invalid_index); - if (RMW_RET_OK != rmw_ret) { - RCL_SET_ERROR_MSG("failed to run check on node name", allocator); - return RCL_RET_ERROR; - } else if (RMW_NODE_NAME_VALID != validation_result) { - RCL_SET_ERROR_MSG_WITH_FORMAT_STRING( - allocator, - "node name prefix invalid: %s", rmw_node_name_validation_result_string(validation_result)); - return RCL_RET_INVALID_REMAP_RULE; - } + ret = rcl_lexer_lookahead2_init(&lex_lookahead, arg, allocator); + if (RCL_RET_OK != ret) { + return ret; } - // Figure out what type of rule this is, default is to apply to topic and service names - rcl_remap_type_t type = RCL_TOPIC_REMAP | RCL_SERVICE_REMAP; - if (4 == len_match && 0 == strncmp("__ns", match_begin, len_match)) { - type = RCL_NAMESPACE_REMAP; - } else if (6 == len_match && 0 == strncmp("__node", match_begin, len_match)) { - type = RCL_NODENAME_REMAP; - } - - if (type & (RCL_TOPIC_REMAP | RCL_SERVICE_REMAP)) { - // Replacement must be a valid topic name - int validation_result; - size_t invalid_index; - rcl_ret_t ret = rcl_validate_topic_name(replacement_begin, &validation_result, &invalid_index); - if (ret != RCL_RET_OK) { - return RCL_RET_ERROR; - } else if (validation_result != RCL_TOPIC_NAME_VALID) { - RCL_SET_ERROR_MSG_WITH_FORMAT_STRING( - allocator, - "replacement is invalid: %s", rcl_topic_name_validation_result_string(validation_result)); - return RCL_RET_INVALID_REMAP_RULE; - } - // Match must be a valid topic name - ret = rcl_validate_topic_name_with_size( - match_begin, len_match, &validation_result, &invalid_index); - if (ret != RCL_RET_OK) { - return RCL_RET_ERROR; - } else if (validation_result != RCL_TOPIC_NAME_VALID) { - RCL_SET_ERROR_MSG_WITH_FORMAT_STRING( - allocator, - "match is invalid: %s", rcl_topic_name_validation_result_string(validation_result)); - return RCL_RET_INVALID_REMAP_RULE; - } - } else if (RCL_NAMESPACE_REMAP == type) { - int validation_result; - size_t invalid_idx; - rmw_ret_t rmw_ret = rmw_validate_namespace(replacement_begin, &validation_result, &invalid_idx); - if (RMW_RET_OK != rmw_ret) { - RCL_SET_ERROR_MSG("failed to run check on namespace", allocator); - return RCL_RET_ERROR; - } else if (RMW_NAMESPACE_VALID != validation_result) { - RCL_SET_ERROR_MSG_WITH_FORMAT_STRING( - allocator, - "namespace is invalid: %s", rmw_namespace_validation_result_string(validation_result)); - return RCL_RET_INVALID_REMAP_RULE; - } - } else if (RCL_NODENAME_REMAP == type) { - int validation_result; - size_t invalid_idx; - rmw_ret_t rmw_ret = rmw_validate_node_name(replacement_begin, &validation_result, &invalid_idx); - if (RMW_RET_OK != rmw_ret) { - RCL_SET_ERROR_MSG("failed to run check on node name", allocator); - return RCL_RET_ERROR; - } else if (RMW_NODE_NAME_VALID != validation_result) { - RCL_SET_ERROR_MSG_WITH_FORMAT_STRING( - allocator, - "node name is invalid: %s", rmw_node_name_validation_result_string(validation_result)); - return RCL_RET_INVALID_REMAP_RULE; - } - } + ret = _rcl_parse_remap_begin_remap_rule(&lex_lookahead, output_rule); - // Rule is valid, construct a structure for it - output_rule->allocator = allocator; - output_rule->type = type; - if (len_node_name > 0) { - output_rule->node_name = rcutils_strndup(arg, len_node_name, allocator); - if (NULL == output_rule->node_name) { - goto cleanup_rule; + if (RCL_RET_OK != ret) { + // cleanup stuff, but return the original error code + if (RCL_RET_OK != rcl_remap_fini(output_rule)) { + RCUTILS_LOG_ERROR_NAMED(ROS_PACKAGE_NAME, "Failed to fini remap rule after error occurred"); } - } - if (type & (RCL_TOPIC_REMAP | RCL_SERVICE_REMAP)) { - output_rule->match = rcutils_strndup(match_begin, len_match, allocator); - if (NULL == output_rule->match) { - goto cleanup_rule; + if (RCL_RET_OK != rcl_lexer_lookahead2_fini(&lex_lookahead)) { + RCUTILS_LOG_ERROR_NAMED(ROS_PACKAGE_NAME, "Failed to fini lookahead2 after error occurred"); } + } else { + ret = rcl_lexer_lookahead2_fini(&lex_lookahead); } - output_rule->replacement = rcutils_strndup(replacement_begin, len_replacement, allocator); - if (NULL == output_rule->replacement) { - goto cleanup_rule; - } - return RCL_RET_OK; - -cleanup_rule: - if (RCL_RET_OK != rcl_remap_fini(output_rule)) { - RCUTILS_LOG_ERROR_NAMED(ROS_PACKAGE_NAME, "Failed to fini remap rule after error occurred"); - } - return RCL_RET_BAD_ALLOC; + return ret; } rcl_ret_t From 12c08120648709315c28c0c9be81b143f89bd8cf Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 09:44:27 -0700 Subject: [PATCH 17/38] Test rosservice:// and rostopic:// --- rcl/test/rcl/test_arguments.cpp | 5 +++++ rcl/test/rcl/test_remap.cpp | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/rcl/test/rcl/test_arguments.cpp b/rcl/test/rcl/test_arguments.cpp index 7a13a61d6..41fdad932 100644 --- a/rcl/test/rcl/test_arguments.cpp +++ b/rcl/test/rcl/test_arguments.cpp @@ -95,6 +95,11 @@ TEST_F(CLASSNAME(TestArgumentsFixture, RMW_IMPLEMENTATION), check_valid_vs_inval EXPECT_TRUE(is_valid_arg("foo:=/bar")); EXPECT_TRUE(is_valid_arg("/foo123:=/bar123")); EXPECT_TRUE(is_valid_arg("node:/foo123:=/bar123")); + EXPECT_TRUE(is_valid_arg("rostopic:=/foo/bar")); + EXPECT_TRUE(is_valid_arg("rosservice:=baz")); + EXPECT_TRUE(is_valid_arg("rostopic://rostopic:=rosservice")); + EXPECT_TRUE(is_valid_arg("rostopic:///rosservice:=rostopic")); + EXPECT_TRUE(is_valid_arg("rostopic:///foo/bar:=baz")); EXPECT_FALSE(is_valid_arg(":=")); EXPECT_FALSE(is_valid_arg("foo:=")); diff --git a/rcl/test/rcl/test_remap.cpp b/rcl/test/rcl/test_remap.cpp index 894c35977..de3835818 100644 --- a/rcl/test/rcl/test_remap.cpp +++ b/rcl/test/rcl/test_remap.cpp @@ -465,3 +465,39 @@ TEST_F(CLASSNAME(TestRemapFixture, RMW_IMPLEMENTATION), other_rules_before_noden EXPECT_STREQ("remap_name", output); allocator.deallocate(output, allocator.state); } + +TEST_F(CLASSNAME(TestRemapFixture, RMW_IMPLEMENTATION), url_scheme_rosservice) { + rcl_ret_t ret; + rcl_arguments_t global_arguments; + SCOPE_ARGS(global_arguments, "process_name", "rosservice://foo:=bar"); + + char * output = NULL; + ret = rcl_remap_service_name( + NULL, &global_arguments, "/ns/foo", "NodeName", "/ns", rcl_get_default_allocator(), &output); + EXPECT_EQ(RCL_RET_OK, ret); + ASSERT_STREQ("/ns/bar", output); + rcl_get_default_allocator().deallocate(output, rcl_get_default_allocator().state); + + ret = rcl_remap_topic_name( + NULL, &global_arguments, "/ns/foo", "NodeName", "/ns", rcl_get_default_allocator(), &output); + EXPECT_EQ(RCL_RET_OK, ret); + EXPECT_EQ(NULL, output); +} + +TEST_F(CLASSNAME(TestRemapFixture, RMW_IMPLEMENTATION), url_scheme_rostopic) { + rcl_ret_t ret; + rcl_arguments_t global_arguments; + SCOPE_ARGS(global_arguments, "process_name", "rostopic://foo:=bar"); + + char * output = NULL; + ret = rcl_remap_topic_name( + NULL, &global_arguments, "/ns/foo", "NodeName", "/ns", rcl_get_default_allocator(), &output); + EXPECT_EQ(RCL_RET_OK, ret); + ASSERT_STREQ("/ns/bar", output); + rcl_get_default_allocator().deallocate(output, rcl_get_default_allocator().state); + + ret = rcl_remap_service_name( + NULL, &global_arguments, "/ns/foo", "NodeName", "/ns", rcl_get_default_allocator(), &output); + EXPECT_EQ(RCL_RET_OK, ret); + EXPECT_EQ(NULL, output); +} From 537e224db902851b6c4e92b79d688db6bb514c85 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 10:56:14 -0700 Subject: [PATCH 18/38] fix movement formula comment --- rcl/src/rcl/lexer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index 0664a5900..dd0f97da1 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -28,7 +28,7 @@ * When a transition is taken it causes the lexer to move to another character in the string. * Normal transitions always move the lexer to the forwards one character. * '' transitions may cause the lexer to move forwards 1, or backwards N. - * The movement M is written as M = 1 - N so it can be stored in an unsigned integer. + * The movement M is written as M = 1 + N so it can be stored in an unsigned integer. * For example, an `` transition with M = 0 moves the lexer forwards 1 character, M = 1 keeps * the lexer at the current character, and M = 2 moves the lexer backwards one character. From a54e669b03b0e065eae8cc7a8ce6e26c874bfd8a Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 13:20:04 -0700 Subject: [PATCH 19/38] doxygent comment --- rcl/include/rcl/lexer.h | 45 +++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/rcl/include/rcl/lexer.h b/rcl/include/rcl/lexer.h index 5c0d77c01..ae61b839e 100644 --- a/rcl/include/rcl/lexer.h +++ b/rcl/include/rcl/lexer.h @@ -27,51 +27,52 @@ extern "C" { #endif +/// Type of lexeme found by lexical analysis. typedef enum rcl_lexeme_t { - // Indicates no valid lexeme was found + /// Indicates no valid lexeme was found RCL_LEXEME_NONE = 0, - // Indicates end of input has been reached + /// Indicates end of input has been reached RCL_LEXEME_EOF = 1, - // ~/ + /// ~/ RCL_LEXEME_TILDE_SLASH = 2, - // rosservice:// + /// rosservice:// RCL_LEXEME_URL_SERVICE = 3, - // rostopic:// + /// rostopic:// RCL_LEXEME_URL_TOPIC = 4, - // : + /// : RCL_LEXEME_COLON = 5, - // __node + /// __node RCL_LEXEME_NODE = 6, - // __ns + /// __ns RCL_LEXEME_NS = 7, - // := + /// := RCL_LEXEME_SEPARATOR = 8, - // \1 + /// \1 RCL_LEXEME_BR1 = 9, - // \2 + /// \2 RCL_LEXEME_BR2 = 10, - // \3 + /// \3 RCL_LEXEME_BR3 = 11, - // \4 + /// \4 RCL_LEXEME_BR4 = 12, - // \5 + /// \5 RCL_LEXEME_BR5 = 13, - // \6 + /// \6 RCL_LEXEME_BR6 = 14, - // \7 + /// \7 RCL_LEXEME_BR7 = 15, - // \8 + /// \8 RCL_LEXEME_BR8 = 16, - // \9 + /// \9 RCL_LEXEME_BR9 = 17, - // a name between slashes, must match (([a-zA-Z](_)?)|_)([0-9a-zA-Z](_)?)* + /// a name between slashes, must match (([a-zA-Z](_)?)|_)([0-9a-zA-Z](_)?)* RCL_LEXEME_TOKEN = 18, - // / + /// / RCL_LEXEME_FORWARD_SLASH = 19, - // * + /// * RCL_LEXEME_WILD_ONE = 20, - // ** + /// ** RCL_LEXEME_WILD_MULTI = 21 } rcl_lexeme_t; From d712f3f591db1543d4a9b073a7e56069457aca63 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 13:25:54 -0700 Subject: [PATCH 20/38] move code to make pr diff easier to read --- rcl/src/rcl/arguments.c | 484 ++++++++++++++++++++-------------------- 1 file changed, 245 insertions(+), 239 deletions(-) diff --git a/rcl/src/rcl/arguments.c b/rcl/src/rcl/arguments.c index 90ce4b990..f32a7c303 100644 --- a/rcl/src/rcl/arguments.c +++ b/rcl/src/rcl/arguments.c @@ -35,6 +35,251 @@ extern "C" // Instance of global arguments. static rcl_arguments_t __rcl_global_arguments; +/// Parse an argument that may or may not be a remap rule. +/// \param[in] arg the argument to parse +/// \param[in] allocator an allocator to use +/// \param[in,out] output_rule input a zero intialized rule, output a fully initialized one +/// \return RCL_RET_OK if a valid rule was parsed, or +/// \return RCL_RET_INVALID_REMAP_RULE if the argument is not a valid rule, or +/// \return RCL_RET_BAD_ALLOC if an allocation failed, or +/// \return RLC_RET_ERROR if an unspecified error occurred. +/// \internal +RCL_LOCAL +rcl_ret_t +_rcl_parse_remap_rule( + const char * arg, + rcl_allocator_t allocator, + rcl_remap_t * output_rule); + +rcl_ret_t +rcl_parse_arguments( + int argc, + const char * const argv[], + rcl_allocator_t allocator, + rcl_arguments_t * args_output) +{ + RCL_CHECK_ALLOCATOR_WITH_MSG(&allocator, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); + if (argc < 0) { + return RCL_RET_INVALID_ARGUMENT; + } else if (argc > 0) { + RCL_CHECK_ARGUMENT_FOR_NULL(argv, RCL_RET_INVALID_ARGUMENT, allocator); + } + RCL_CHECK_ARGUMENT_FOR_NULL(args_output, RCL_RET_INVALID_ARGUMENT, allocator); + + rcl_ret_t ret; + rcl_ret_t fail_ret; + + args_output->impl = allocator.allocate(sizeof(rcl_arguments_impl_t), allocator.state); + if (NULL == args_output->impl) { + return RCL_RET_BAD_ALLOC; + } + rcl_arguments_impl_t * args_impl = args_output->impl; + args_impl->num_remap_rules = 0; + args_impl->remap_rules = NULL; + args_impl->unparsed_args = NULL; + args_impl->num_unparsed_args = 0; + args_impl->allocator = allocator; + + if (argc == 0) { + // there are no arguments to parse + return RCL_RET_OK; + } + + // over-allocate arrays to match the number of arguments + args_impl->remap_rules = allocator.allocate(sizeof(rcl_remap_t) * argc, allocator.state); + if (NULL == args_impl->remap_rules) { + ret = RCL_RET_BAD_ALLOC; + goto fail; + } + args_impl->unparsed_args = allocator.allocate(sizeof(int) * argc, allocator.state); + if (NULL == args_impl->unparsed_args) { + ret = RCL_RET_BAD_ALLOC; + goto fail; + } + + // Attempt to parse arguments as remap rules + for (int i = 0; i < argc; ++i) { + rcl_remap_t * rule = &(args_impl->remap_rules[args_impl->num_remap_rules]); + *rule = rcl_remap_get_zero_initialized(); + if (RCL_RET_OK == _rcl_parse_remap_rule(argv[i], allocator, rule)) { + ++(args_impl->num_remap_rules); + } else { + RCUTILS_LOG_DEBUG_NAMED(ROS_PACKAGE_NAME, "arg %d error '%s'", i, rcl_get_error_string()); + rcl_reset_error(); + args_impl->unparsed_args[args_impl->num_unparsed_args] = i; + ++(args_impl->num_unparsed_args); + } + } + + // Shrink remap_rules array to match number of successfully parsed rules + if (args_impl->num_remap_rules > 0) { + args_impl->remap_rules = rcutils_reallocf( + args_impl->remap_rules, sizeof(rcl_remap_t) * args_impl->num_remap_rules, &allocator); + if (NULL == args_impl->remap_rules) { + ret = RCL_RET_BAD_ALLOC; + goto fail; + } + } else { + // No remap rules + allocator.deallocate(args_impl->remap_rules, allocator.state); + args_impl->remap_rules = NULL; + } + // Shrink unparsed_args + if (0 == args_impl->num_unparsed_args) { + // No unparsed args + allocator.deallocate(args_impl->unparsed_args, allocator.state); + args_impl->unparsed_args = NULL; + } else if (args_impl->num_unparsed_args < argc) { + args_impl->unparsed_args = rcutils_reallocf( + args_impl->unparsed_args, sizeof(int) * args_impl->num_unparsed_args, &allocator); + if (NULL == args_impl->unparsed_args) { + ret = RCL_RET_BAD_ALLOC; + goto fail; + } + } + + return RCL_RET_OK; +fail: + fail_ret = ret; + if (NULL != args_impl) { + ret = rcl_arguments_fini(args_output); + if (RCL_RET_OK != ret) { + RCUTILS_LOG_ERROR_NAMED(ROS_PACKAGE_NAME, "Failed to fini arguments after earlier failure"); + } + } + return fail_ret; +} + +int +rcl_arguments_get_count_unparsed( + const rcl_arguments_t * args) +{ + if (NULL == args || NULL == args->impl) { + return -1; + } + return args->impl->num_unparsed_args; +} + +rcl_ret_t +rcl_arguments_get_unparsed( + const rcl_arguments_t * args, + rcl_allocator_t allocator, + int ** output_unparsed_indices) +{ + RCL_CHECK_ALLOCATOR_WITH_MSG(&allocator, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); + RCL_CHECK_ARGUMENT_FOR_NULL(args, RCL_RET_INVALID_ARGUMENT, allocator); + RCL_CHECK_ARGUMENT_FOR_NULL(args->impl, RCL_RET_INVALID_ARGUMENT, allocator); + RCL_CHECK_ARGUMENT_FOR_NULL(output_unparsed_indices, RCL_RET_INVALID_ARGUMENT, allocator); + + *output_unparsed_indices = NULL; + if (args->impl->num_unparsed_args) { + *output_unparsed_indices = allocator.allocate( + sizeof(int) * args->impl->num_unparsed_args, allocator.state); + if (NULL == *output_unparsed_indices) { + return RCL_RET_BAD_ALLOC; + } + for (int i = 0; i < args->impl->num_unparsed_args; ++i) { + (*output_unparsed_indices)[i] = args->impl->unparsed_args[i]; + } + } + return RCL_RET_OK; +} + +rcl_arguments_t +rcl_get_zero_initialized_arguments(void) +{ + static rcl_arguments_t default_arguments = { + .impl = NULL + }; + return default_arguments; +} + +rcl_ret_t +rcl_remove_ros_arguments( + char const * const argv[], + const rcl_arguments_t * args, + rcl_allocator_t allocator, + int * nonros_argc, + const char ** nonros_argv[]) +{ + RCL_CHECK_ALLOCATOR_WITH_MSG(&allocator, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); + RCL_CHECK_ARGUMENT_FOR_NULL(argv, RCL_RET_INVALID_ARGUMENT, allocator); + RCL_CHECK_ARGUMENT_FOR_NULL(nonros_argc, RCL_RET_INVALID_ARGUMENT, allocator); + RCL_CHECK_ARGUMENT_FOR_NULL(args, RCL_RET_INVALID_ARGUMENT, allocator); + + *nonros_argc = rcl_arguments_get_count_unparsed(args); + *nonros_argv = NULL; + + if (*nonros_argc <= 0) { + return RCL_RET_INVALID_ARGUMENT; + } + + int * unparsed_indices = NULL; + rcl_ret_t ret; + ret = rcl_arguments_get_unparsed(args, allocator, &unparsed_indices); + + if (RCL_RET_OK != ret) { + return ret; + } + + size_t alloc_size = sizeof(char *) * *nonros_argc; + *nonros_argv = allocator.allocate(alloc_size, allocator.state); + if (NULL == *nonros_argv) { + allocator.deallocate(unparsed_indices, allocator.state); + return RCL_RET_BAD_ALLOC; + } + for (int i = 0; i < *nonros_argc; ++i) { + (*nonros_argv)[i] = argv[unparsed_indices[i]]; + } + + allocator.deallocate(unparsed_indices, allocator.state); + return RCL_RET_OK; +} + +rcl_ret_t +rcl_arguments_fini( + rcl_arguments_t * args) +{ + rcl_allocator_t alloc = rcl_get_default_allocator(); + RCL_CHECK_ARGUMENT_FOR_NULL(args, RCL_RET_INVALID_ARGUMENT, alloc); + if (args->impl) { + rcl_ret_t ret = RCL_RET_OK; + alloc = args->impl->allocator; + if (args->impl->remap_rules) { + for (int i = 0; i < args->impl->num_remap_rules; ++i) { + rcl_ret_t remap_ret = rcl_remap_fini(&(args->impl->remap_rules[i])); + if (remap_ret != RCL_RET_OK) { + ret = remap_ret; + RCUTILS_LOG_ERROR_NAMED( + ROS_PACKAGE_NAME, + "Failed to finalize remap rule while finalizing arguments. Continuing..."); + } + } + args->impl->allocator.deallocate(args->impl->remap_rules, args->impl->allocator.state); + args->impl->remap_rules = NULL; + args->impl->num_remap_rules = 0; + } + + args->impl->allocator.deallocate(args->impl->unparsed_args, args->impl->allocator.state); + args->impl->num_unparsed_args = 0; + args->impl->unparsed_args = NULL; + + args->impl->allocator.deallocate(args->impl, args->impl->allocator.state); + args->impl = NULL; + return ret; + } + RCL_SET_ERROR_MSG("rcl_arguments_t finalized twice", alloc); + return RCL_RET_ERROR; +} + +RCL_PUBLIC +RCL_WARN_UNUSED +rcl_arguments_t * +rcl_get_global_arguments() +{ + return &__rcl_global_arguments; +} + /// Parses a fully qualified namespace for a namespace replacement rule (ex: `/foo/bar`) /// \sa _rcl_parse_remap_begin_remap_rule() /// \internal @@ -490,16 +735,6 @@ _rcl_parse_remap_begin_remap_rule( return ret; } -/// Parse an argument that may or may not be a remap rule. -/// \param[in] arg the argument to parse -/// \param[in] allocator an allocator to use -/// \param[in,out] output_rule input a zero intialized rule, output a fully initialized one -/// \return RCL_RET_OK if a valid rule was parsed, or -/// \return RCL_RET_INVALID_REMAP_RULE if the argument is not a valid rule, or -/// \return RCL_RET_BAD_ALLOC if an allocation failed, or -/// \return RLC_RET_ERROR if an unspecified error occurred. -/// \internal -RCL_LOCAL rcl_ret_t _rcl_parse_remap_rule( const char * arg, @@ -535,235 +770,6 @@ _rcl_parse_remap_rule( return ret; } -rcl_ret_t -rcl_parse_arguments( - int argc, - const char * const argv[], - rcl_allocator_t allocator, - rcl_arguments_t * args_output) -{ - RCL_CHECK_ALLOCATOR_WITH_MSG(&allocator, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); - if (argc < 0) { - return RCL_RET_INVALID_ARGUMENT; - } else if (argc > 0) { - RCL_CHECK_ARGUMENT_FOR_NULL(argv, RCL_RET_INVALID_ARGUMENT, allocator); - } - RCL_CHECK_ARGUMENT_FOR_NULL(args_output, RCL_RET_INVALID_ARGUMENT, allocator); - - rcl_ret_t ret; - rcl_ret_t fail_ret; - - args_output->impl = allocator.allocate(sizeof(rcl_arguments_impl_t), allocator.state); - if (NULL == args_output->impl) { - return RCL_RET_BAD_ALLOC; - } - rcl_arguments_impl_t * args_impl = args_output->impl; - args_impl->num_remap_rules = 0; - args_impl->remap_rules = NULL; - args_impl->unparsed_args = NULL; - args_impl->num_unparsed_args = 0; - args_impl->allocator = allocator; - - if (argc == 0) { - // there are no arguments to parse - return RCL_RET_OK; - } - - // over-allocate arrays to match the number of arguments - args_impl->remap_rules = allocator.allocate(sizeof(rcl_remap_t) * argc, allocator.state); - if (NULL == args_impl->remap_rules) { - ret = RCL_RET_BAD_ALLOC; - goto fail; - } - args_impl->unparsed_args = allocator.allocate(sizeof(int) * argc, allocator.state); - if (NULL == args_impl->unparsed_args) { - ret = RCL_RET_BAD_ALLOC; - goto fail; - } - - // Attempt to parse arguments as remap rules - for (int i = 0; i < argc; ++i) { - rcl_remap_t * rule = &(args_impl->remap_rules[args_impl->num_remap_rules]); - *rule = rcl_remap_get_zero_initialized(); - if (RCL_RET_OK == _rcl_parse_remap_rule(argv[i], allocator, rule)) { - ++(args_impl->num_remap_rules); - } else { - RCUTILS_LOG_DEBUG_NAMED(ROS_PACKAGE_NAME, "arg %d error '%s'", i, rcl_get_error_string()); - rcl_reset_error(); - args_impl->unparsed_args[args_impl->num_unparsed_args] = i; - ++(args_impl->num_unparsed_args); - } - } - - // Shrink remap_rules array to match number of successfully parsed rules - if (args_impl->num_remap_rules > 0) { - args_impl->remap_rules = rcutils_reallocf( - args_impl->remap_rules, sizeof(rcl_remap_t) * args_impl->num_remap_rules, &allocator); - if (NULL == args_impl->remap_rules) { - ret = RCL_RET_BAD_ALLOC; - goto fail; - } - } else { - // No remap rules - allocator.deallocate(args_impl->remap_rules, allocator.state); - args_impl->remap_rules = NULL; - } - // Shrink unparsed_args - if (0 == args_impl->num_unparsed_args) { - // No unparsed args - allocator.deallocate(args_impl->unparsed_args, allocator.state); - args_impl->unparsed_args = NULL; - } else if (args_impl->num_unparsed_args < argc) { - args_impl->unparsed_args = rcutils_reallocf( - args_impl->unparsed_args, sizeof(int) * args_impl->num_unparsed_args, &allocator); - if (NULL == args_impl->unparsed_args) { - ret = RCL_RET_BAD_ALLOC; - goto fail; - } - } - - return RCL_RET_OK; -fail: - fail_ret = ret; - if (NULL != args_impl) { - ret = rcl_arguments_fini(args_output); - if (RCL_RET_OK != ret) { - RCUTILS_LOG_ERROR_NAMED(ROS_PACKAGE_NAME, "Failed to fini arguments after earlier failure"); - } - } - return fail_ret; -} - -int -rcl_arguments_get_count_unparsed( - const rcl_arguments_t * args) -{ - if (NULL == args || NULL == args->impl) { - return -1; - } - return args->impl->num_unparsed_args; -} - -rcl_ret_t -rcl_arguments_get_unparsed( - const rcl_arguments_t * args, - rcl_allocator_t allocator, - int ** output_unparsed_indices) -{ - RCL_CHECK_ALLOCATOR_WITH_MSG(&allocator, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); - RCL_CHECK_ARGUMENT_FOR_NULL(args, RCL_RET_INVALID_ARGUMENT, allocator); - RCL_CHECK_ARGUMENT_FOR_NULL(args->impl, RCL_RET_INVALID_ARGUMENT, allocator); - RCL_CHECK_ARGUMENT_FOR_NULL(output_unparsed_indices, RCL_RET_INVALID_ARGUMENT, allocator); - - *output_unparsed_indices = NULL; - if (args->impl->num_unparsed_args) { - *output_unparsed_indices = allocator.allocate( - sizeof(int) * args->impl->num_unparsed_args, allocator.state); - if (NULL == *output_unparsed_indices) { - return RCL_RET_BAD_ALLOC; - } - for (int i = 0; i < args->impl->num_unparsed_args; ++i) { - (*output_unparsed_indices)[i] = args->impl->unparsed_args[i]; - } - } - return RCL_RET_OK; -} - -rcl_arguments_t -rcl_get_zero_initialized_arguments(void) -{ - static rcl_arguments_t default_arguments = { - .impl = NULL - }; - return default_arguments; -} - -rcl_ret_t -rcl_remove_ros_arguments( - char const * const argv[], - const rcl_arguments_t * args, - rcl_allocator_t allocator, - int * nonros_argc, - const char ** nonros_argv[]) -{ - RCL_CHECK_ALLOCATOR_WITH_MSG(&allocator, "invalid allocator", return RCL_RET_INVALID_ARGUMENT); - RCL_CHECK_ARGUMENT_FOR_NULL(argv, RCL_RET_INVALID_ARGUMENT, allocator); - RCL_CHECK_ARGUMENT_FOR_NULL(nonros_argc, RCL_RET_INVALID_ARGUMENT, allocator); - RCL_CHECK_ARGUMENT_FOR_NULL(args, RCL_RET_INVALID_ARGUMENT, allocator); - - *nonros_argc = rcl_arguments_get_count_unparsed(args); - *nonros_argv = NULL; - - if (*nonros_argc <= 0) { - return RCL_RET_INVALID_ARGUMENT; - } - - int * unparsed_indices = NULL; - rcl_ret_t ret; - ret = rcl_arguments_get_unparsed(args, allocator, &unparsed_indices); - - if (RCL_RET_OK != ret) { - return ret; - } - - size_t alloc_size = sizeof(char *) * *nonros_argc; - *nonros_argv = allocator.allocate(alloc_size, allocator.state); - if (NULL == *nonros_argv) { - allocator.deallocate(unparsed_indices, allocator.state); - return RCL_RET_BAD_ALLOC; - } - for (int i = 0; i < *nonros_argc; ++i) { - (*nonros_argv)[i] = argv[unparsed_indices[i]]; - } - - allocator.deallocate(unparsed_indices, allocator.state); - return RCL_RET_OK; -} - -rcl_ret_t -rcl_arguments_fini( - rcl_arguments_t * args) -{ - rcl_allocator_t alloc = rcl_get_default_allocator(); - RCL_CHECK_ARGUMENT_FOR_NULL(args, RCL_RET_INVALID_ARGUMENT, alloc); - if (args->impl) { - rcl_ret_t ret = RCL_RET_OK; - alloc = args->impl->allocator; - if (args->impl->remap_rules) { - for (int i = 0; i < args->impl->num_remap_rules; ++i) { - rcl_ret_t remap_ret = rcl_remap_fini(&(args->impl->remap_rules[i])); - if (remap_ret != RCL_RET_OK) { - ret = remap_ret; - RCUTILS_LOG_ERROR_NAMED( - ROS_PACKAGE_NAME, - "Failed to finalize remap rule while finalizing arguments. Continuing..."); - } - } - args->impl->allocator.deallocate(args->impl->remap_rules, args->impl->allocator.state); - args->impl->remap_rules = NULL; - args->impl->num_remap_rules = 0; - } - - args->impl->allocator.deallocate(args->impl->unparsed_args, args->impl->allocator.state); - args->impl->num_unparsed_args = 0; - args->impl->unparsed_args = NULL; - - args->impl->allocator.deallocate(args->impl, args->impl->allocator.state); - args->impl = NULL; - return ret; - } - RCL_SET_ERROR_MSG("rcl_arguments_t finalized twice", alloc); - return RCL_RET_ERROR; -} - -RCL_PUBLIC -RCL_WARN_UNUSED -rcl_arguments_t * -rcl_get_global_arguments() -{ - return &__rcl_global_arguments; -} - #if __cplusplus } #endif From d6c63136ef79999794e09767b51b3c294d20e585 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 13:32:12 -0700 Subject: [PATCH 21/38] Comments --- rcl/src/rcl/lexer.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index dd0f97da1..50434f9d3 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -19,14 +19,14 @@ * It looks at one character at a time, and uses that character's value to decide how to transition * a state machine. * A transition is taken if a character's ASCII value falls within its range. - * No transition's ranges overlap; there is never more than one matching transition. + * There is never more than one matching transition. * - * If no transition matches then the state machine takes an '' transition. + * If no transition matches then it uses a state's '' transition. * Every state has exactly one '' transition. - * All states have an `` to T_NONE unless otherwise specified in the diagram below. + * In the diagram below all states have an `` to T_NONE unless otherwise specified. * * When a transition is taken it causes the lexer to move to another character in the string. - * Normal transitions always move the lexer to the forwards one character. + * Normal transitions always move the lexer forwards one character. * '' transitions may cause the lexer to move forwards 1, or backwards N. * The movement M is written as M = 1 + N so it can be stored in an unsigned integer. * For example, an `` transition with M = 0 moves the lexer forwards 1 character, M = 1 keeps @@ -134,25 +134,27 @@ digraph remapping_lexer { } */ -// Represents a transition from one state to another +/// Represents a transition from one state to another +/// \internal typedef struct rcl_lexer_transition_t { - // Index of a state to transition to + /// Index of a state to transition to const unsigned char to_state; - // Start of a range of chars (inclusive) which activates this transition + /// Start of a range of chars (inclusive) which activates this transition const char range_start; - // End of a range of chars (inclusive) which activates this transition + /// End of a range of chars (inclusive) which activates this transition const char range_end; } rcl_lexer_transition_t; -// Represents a non-terminal state +/// Represents a non-terminal state +/// \internal typedef struct rcl_lexer_state_t { - // If no transition matches this causes a character to be analyzed a second time in another state + /// If no transition matches this causes a character to be analyzed a second time in another state const unsigned char else_state; - // Movement associated with taking else state + /// Movement associated with taking else state const unsigned char else_movement; - // Transitions in the state machine (NULL value at end of array) + /// Transitions in the state machine (NULL value at end of array) const rcl_lexer_transition_t transitions[11]; } rcl_lexer_state_t; From cf2b32895580b0ea427f623a61aff3f665c6ae5f Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 13:34:39 -0700 Subject: [PATCH 22/38] Comment --- rcl/src/rcl/lexer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index 50434f9d3..ac6915d25 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -150,7 +150,7 @@ typedef struct rcl_lexer_transition_t /// \internal typedef struct rcl_lexer_state_t { - /// If no transition matches this causes a character to be analyzed a second time in another state + /// Transition to this state if no other transition matches const unsigned char else_state; /// Movement associated with taking else state const unsigned char else_movement; From 2867c74299da7ea396fa43c80d2b772eee50f519 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 13:51:50 -0700 Subject: [PATCH 23/38] Comment about impossibilities --- rcl/src/rcl/lexer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index ac6915d25..b54632472 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -603,6 +603,7 @@ rcl_lexer_analyze( // Analyze one character at a time until lexeme is found do { if (next_state > LAST_STATE) { + // Should never happen RCL_SET_ERROR_MSG("Internal lexer bug: next state does not exist", alloc); return RCL_RET_ERROR; } @@ -636,6 +637,7 @@ rcl_lexer_analyze( } else { // Go backwards N chars if (movement - 1 > *length) { + // Should never happen RCL_SET_ERROR_MSG("Internal lexer bug: movement would read before start of string", alloc); return RCL_RET_ERROR; } @@ -644,6 +646,7 @@ rcl_lexer_analyze( } while (next_state < FIRST_TERMINAL); if (FIRST_TERMINAL > next_state || next_state - FIRST_TERMINAL > LAST_TERMINAL) { + // Should never happen RCL_SET_ERROR_MSG("Internal lexer bug: terminal state does not exist", alloc); return RCL_RET_ERROR; } From a9ee9be1b178a80bebc0b766d7789dad5509ab27 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:00:07 -0700 Subject: [PATCH 24/38] Set error message --- rcl/src/rcl/arguments.c | 1 + 1 file changed, 1 insertion(+) diff --git a/rcl/src/rcl/arguments.c b/rcl/src/rcl/arguments.c index f32a7c303..f319e3749 100644 --- a/rcl/src/rcl/arguments.c +++ b/rcl/src/rcl/arguments.c @@ -437,6 +437,7 @@ _rcl_parse_remap_match_token( RCL_SET_ERROR_MSG("Wildcard '**' is not implemented", rule->allocator); return RCL_RET_ERROR; } else { + RCL_SET_ERROR_MSG("Expecting token or wildcard", rule->allocator); ret = RCL_RET_INVALID_REMAP_RULE; } From 227743a387bc180051fba4016845c0b486e031d7 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:00:24 -0700 Subject: [PATCH 25/38] unsigned literals --- rcl/src/rcl/lexer.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rcl/src/rcl/lexer.c b/rcl/src/rcl/lexer.c index b54632472..6bda5af3f 100644 --- a/rcl/src/rcl/lexer.c +++ b/rcl/src/rcl/lexer.c @@ -587,9 +587,9 @@ rcl_lexer_analyze( RCL_CHECK_ARGUMENT_FOR_NULL(lexeme, RCL_RET_INVALID_ARGUMENT, alloc); RCL_CHECK_ARGUMENT_FOR_NULL(length, RCL_RET_INVALID_ARGUMENT, alloc); - *length = 0; + *length = 0u; - if ('\0' == text[0]) { + if ('\0' == text[0u]) { // Early exit if string is empty *lexeme = RCL_LEXEME_EOF; return RCL_RET_OK; @@ -609,11 +609,11 @@ rcl_lexer_analyze( } state = &(g_states[next_state]); current_char = text[*length]; - next_state = 0; - movement = 0; + next_state = 0u; + movement = 0u; // Look for a transition that contains this character in its range - size_t transition_idx = 0; + size_t transition_idx = 0u; const rcl_lexer_transition_t * transition; do { transition = &(state->transitions[transition_idx]); @@ -622,26 +622,26 @@ rcl_lexer_analyze( break; } ++transition_idx; - } while (transition->to_state != 0); + } while (0u != transition->to_state); // if no transition was found, take the else transition - if (0 == next_state) { + if (0u == next_state) { next_state = state->else_state; movement = state->else_movement; } // Move the lexer to another character in the string - if (0 == movement) { + if (0u == movement) { // Go forwards 1 char ++(*length); } else { // Go backwards N chars - if (movement - 1 > *length) { + if (movement - 1u > *length) { // Should never happen RCL_SET_ERROR_MSG("Internal lexer bug: movement would read before start of string", alloc); return RCL_RET_ERROR; } - *length -= movement - 1; + *length -= movement - 1u; } } while (next_state < FIRST_TERMINAL); From 23232ec40edb14589be446fbfc3c4e46bb21ff43 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:09:21 -0700 Subject: [PATCH 26/38] Add a couple more url scheme tests --- rcl/test/rcl/test_arguments.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rcl/test/rcl/test_arguments.cpp b/rcl/test/rcl/test_arguments.cpp index 41fdad932..91df159e7 100644 --- a/rcl/test/rcl/test_arguments.cpp +++ b/rcl/test/rcl/test_arguments.cpp @@ -115,6 +115,8 @@ TEST_F(CLASSNAME(TestArgumentsFixture, RMW_IMPLEMENTATION), check_valid_vs_inval EXPECT_FALSE(is_valid_arg("foo:=/b ar")); EXPECT_FALSE(is_valid_arg("f{oo:=/bar")); EXPECT_FALSE(is_valid_arg("foo:=/b}ar")); + EXPECT_FALSE(is_valid_arg("rostopic://:=rosservice")); + EXPECT_FALSE(is_valid_arg("rostopic::=rosservice")); } TEST_F(CLASSNAME(TestArgumentsFixture, RMW_IMPLEMENTATION), test_no_args) { From 4ce994fee851e81070a0421c87a4ff93939a6d79 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:12:00 -0700 Subject: [PATCH 27/38] remove out of date comment --- rcl/test/rcl/test_lexer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index cb59c6bc5..dbca9ba84 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -16,7 +16,6 @@ #include -// test include directory has src #include "rcl/lexer.h" #ifdef RMW_IMPLEMENTATION From 413b720ef41e59d1217711c496939ad8c6917ab8 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:14:07 -0700 Subject: [PATCH 28/38] end_pos -> length --- rcl/test/rcl/test_lexer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index dbca9ba84..c1a1ede34 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -41,12 +41,12 @@ class CLASSNAME (TestLexerFixture, RMW_IMPLEMENTATION) : public ::testing::Test #define EXPECT_LEX(expected_lexeme, expected_text, text) \ do { \ rcl_lexeme_t actual_lexeme; \ - size_t end_pos; \ + size_t length; \ rcl_allocator_t allocator = rcl_get_default_allocator(); \ - rcl_ret_t ret = rcl_lexer_analyze(text, allocator, &actual_lexeme, &end_pos); \ + rcl_ret_t ret = rcl_lexer_analyze(text, allocator, &actual_lexeme, &length); \ ASSERT_EQ(RCL_RET_OK, ret); \ EXPECT_EQ(expected_lexeme, actual_lexeme); \ - std::string actual_text(text, end_pos); \ + std::string actual_text(text, length); \ EXPECT_STREQ(expected_text, actual_text.c_str()); \ } while (false) From 19b64a101c5fcf88215ffe0d2877be0e7b9c714a Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:15:13 -0700 Subject: [PATCH 29/38] another token text --- rcl/test/rcl/test_lexer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index c1a1ede34..cdd858dd1 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -55,6 +55,8 @@ TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_token) // Things get recognized as tokens whether input ends or non token characters come after them EXPECT_LEX(RCL_LEXEME_TOKEN, "foo", "foo"); EXPECT_LEX(RCL_LEXEME_TOKEN, "foo", "foo:"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "foo_", "foo_"); + EXPECT_LEX(RCL_LEXEME_TOKEN, "foo_", "foo_:"); // Check full range for starting character EXPECT_LEX(RCL_LEXEME_TOKEN, "a", "a"); From 593133e14beea3ee2296a84d2022b8d869b93ead Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:21:37 -0700 Subject: [PATCH 30/38] Whitespace --- rcl/test/rcl/test_lexer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/rcl/test/rcl/test_lexer.cpp b/rcl/test/rcl/test_lexer.cpp index cdd858dd1..26a6dc5e7 100644 --- a/rcl/test/rcl/test_lexer.cpp +++ b/rcl/test/rcl/test_lexer.cpp @@ -118,6 +118,7 @@ TEST_F(CLASSNAME(TestLexerFixture, RMW_IMPLEMENTATION), test_token) EXPECT_LEX(RCL_LEXEME_NONE, "[", "["); EXPECT_LEX(RCL_LEXEME_NONE, "`", "`"); EXPECT_LEX(RCL_LEXEME_NONE, "{", "{"); + // Tokens cannot start with digits EXPECT_LEX(RCL_LEXEME_NONE, "0", "0"); EXPECT_LEX(RCL_LEXEME_NONE, "1", "1"); From fb5e149d78e14f1dfd3a0844f5bed235189dd73f Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:32:06 -0700 Subject: [PATCH 31/38] call accept -> accept --- rcl/include/rcl/lexer_lookahead.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rcl/include/rcl/lexer_lookahead.h b/rcl/include/rcl/lexer_lookahead.h index 14c5b34a9..a72f80b86 100644 --- a/rcl/include/rcl/lexer_lookahead.h +++ b/rcl/include/rcl/lexer_lookahead.h @@ -142,7 +142,7 @@ rcl_lexer_lookahead2_peek( /// Look ahead at the next two lexemes in the string. /** * Repeated calls to peek2 will return the same two lexemes. - * A parser that deems the next two lexemes as valid must call accept twice to advance lexing. + * A parser that deems the next two lexemes as valid must accept twice to advance lexing. * \sa rcl_lexer_lookahead2_accept() * *
From 05cd29576215f8639c7f7e133b890ba4d121115c Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 14:53:00 -0700 Subject: [PATCH 32/38] use array instead of number suffix --- rcl/src/rcl/lexer_lookahead.c | 75 ++++++++++++++++------------------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/rcl/src/rcl/lexer_lookahead.c b/rcl/src/rcl/lexer_lookahead.c index 34f849d4a..fc389097b 100644 --- a/rcl/src/rcl/lexer_lookahead.c +++ b/rcl/src/rcl/lexer_lookahead.c @@ -22,19 +22,12 @@ struct rcl_lexer_lookahead2_impl_t // Where in the text analysis is being performed size_t text_idx; - // index of first character of first terminal in buffer - size_t start1; - // One past last character of first terminal in buffer - size_t end1; - // Type of first terminal in buffer - rcl_lexeme_t type1; - - // index of first character of second terminal in buffer - size_t start2; - // One past last character of second terminal in buffer - size_t end2; - // Type of second terminal in buffer - rcl_lexeme_t type2; + // first character of lexeme + size_t start[2]; + // One past last character of lexeme + size_t end[2]; + // Type of lexeme + rcl_lexeme_t type[2]; // Allocator to use if an error occurrs rcl_allocator_t allocator; @@ -68,13 +61,13 @@ rcl_lexer_lookahead2_init( buffer->impl, "Failed to allocate lookahead impl", return RCL_RET_BAD_ALLOC, allocator); buffer->impl->text = text; - buffer->impl->text_idx = 0; - buffer->impl->start1 = 0; - buffer->impl->end1 = 0; - buffer->impl->type1 = RCL_LEXEME_NONE; - buffer->impl->start2 = 0; - buffer->impl->end2 = 0; - buffer->impl->type2 = RCL_LEXEME_NONE; + buffer->impl->text_idx = 0u; + buffer->impl->start[0] = 0u; + buffer->impl->start[1] = 0u; + buffer->impl->end[0] = 0u; + buffer->impl->end[1] = 0u; + buffer->impl->type[0] = RCL_LEXEME_NONE; + buffer->impl->type[1] = RCL_LEXEME_NONE; buffer->impl->allocator = allocator; return RCL_RET_OK; @@ -110,23 +103,23 @@ rcl_lexer_lookahead2_peek( rcl_ret_t ret; size_t length; - if (buffer->impl->text_idx >= buffer->impl->end1) { + if (buffer->impl->text_idx >= buffer->impl->end[0]) { // No buffered lexeme; get one ret = rcl_lexer_analyze( rcl_lexer_lookahead2_get_text(buffer), buffer->impl->allocator, - &(buffer->impl->type1), + &(buffer->impl->type[0]), &length); if (RCL_RET_OK != ret) { return ret; } - buffer->impl->start1 = buffer->impl->text_idx; - buffer->impl->end1 = buffer->impl->start1 + length; + buffer->impl->start[0] = buffer->impl->text_idx; + buffer->impl->end[0] = buffer->impl->start[0] + length; } - *next_type = buffer->impl->type1; + *next_type = buffer->impl->type[0]; return RCL_RET_OK; } @@ -137,7 +130,7 @@ rcl_lexer_lookahead2_peek2( rcl_lexeme_t * next_type2) { rcl_ret_t ret; - // Peek 1 ahead first (reusing its error checking for buffer and next_type1) + // Peek 1 ahead first (reusing its error checking for buffer and next_type[0]) ret = rcl_lexer_lookahead2_peek(buffer, next_type1); if (RCL_RET_OK != ret) { return ret; @@ -146,23 +139,23 @@ rcl_lexer_lookahead2_peek2( size_t length; - if (buffer->impl->text_idx >= buffer->impl->end2) { + if (buffer->impl->text_idx >= buffer->impl->end[1]) { // No buffered lexeme; get one ret = rcl_lexer_analyze( - &(buffer->impl->text[buffer->impl->end1]), + &(buffer->impl->text[buffer->impl->end[0]]), buffer->impl->allocator, - &(buffer->impl->type2), + &(buffer->impl->type[1]), &length); if (RCL_RET_OK != ret) { return ret; } - buffer->impl->start2 = buffer->impl->end1; - buffer->impl->end2 = buffer->impl->start2 + length; + buffer->impl->start[1] = buffer->impl->end[0]; + buffer->impl->end[1] = buffer->impl->start[1] + length; } - *next_type2 = buffer->impl->type2; + *next_type2 = buffer->impl->type[1]; return RCL_RET_OK; } @@ -183,32 +176,32 @@ rcl_lexer_lookahead2_accept( RCL_SET_ERROR_MSG("text and length must both be set or both be NULL", buffer->impl->allocator); } - if (RCL_LEXEME_EOF == buffer->impl->type1) { + if (RCL_LEXEME_EOF == buffer->impl->type[0]) { // Reached EOF, nothing to accept if (NULL != lexeme_text && NULL != lexeme_text_length) { *lexeme_text = rcl_lexer_lookahead2_get_text(buffer); - *lexeme_text_length = 0; + *lexeme_text_length = 0u; } return RCL_RET_OK; } - if (buffer->impl->text_idx >= buffer->impl->end1) { + if (buffer->impl->text_idx >= buffer->impl->end[0]) { RCL_SET_ERROR_MSG("no lexeme to accept", buffer->impl->allocator); return RCL_RET_ERROR; } if (NULL != lexeme_text && NULL != lexeme_text_length) { - *lexeme_text = &(buffer->impl->text[buffer->impl->start1]); - *lexeme_text_length = buffer->impl->end1 - buffer->impl->start1; + *lexeme_text = &(buffer->impl->text[buffer->impl->start[0]]); + *lexeme_text_length = buffer->impl->end[0] - buffer->impl->start[0]; } // Advance lexer position - buffer->impl->text_idx = buffer->impl->end1; + buffer->impl->text_idx = buffer->impl->end[0]; // Move second lexeme in buffer to first position - buffer->impl->start1 = buffer->impl->start2; - buffer->impl->end1 = buffer->impl->end2; - buffer->impl->type1 = buffer->impl->type2; + buffer->impl->start[0] = buffer->impl->start[1]; + buffer->impl->end[0] = buffer->impl->end[1]; + buffer->impl->type[0] = buffer->impl->type[1]; return RCL_RET_OK; } From 591c35fee10616b75ca1c31a7b1e77425a1c9ead Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 15:07:27 -0700 Subject: [PATCH 33/38] Missing return; wrong comment --- rcl/src/rcl/lexer_lookahead.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rcl/src/rcl/lexer_lookahead.c b/rcl/src/rcl/lexer_lookahead.c index fc389097b..f2c882152 100644 --- a/rcl/src/rcl/lexer_lookahead.c +++ b/rcl/src/rcl/lexer_lookahead.c @@ -130,7 +130,7 @@ rcl_lexer_lookahead2_peek2( rcl_lexeme_t * next_type2) { rcl_ret_t ret; - // Peek 1 ahead first (reusing its error checking for buffer and next_type[0]) + // Peek 1 ahead first (reusing its error checking for buffer and next_type1) ret = rcl_lexer_lookahead2_peek(buffer, next_type1); if (RCL_RET_OK != ret) { return ret; @@ -174,6 +174,7 @@ rcl_lexer_lookahead2_accept( (NULL != lexeme_text && NULL == lexeme_text_length)) { RCL_SET_ERROR_MSG("text and length must both be set or both be NULL", buffer->impl->allocator); + return RCL_RET_INVALID_ARGUMENT; } if (RCL_LEXEME_EOF == buffer->impl->type[0]) { From a4ffdc27ec77d2a30d713944aca5ae9e8d6d3257 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 9 Apr 2018 15:22:51 -0700 Subject: [PATCH 34/38] test methods at end of input --- rcl/test/rcl/test_lexer_lookahead.cpp | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/rcl/test/rcl/test_lexer_lookahead.cpp b/rcl/test/rcl/test_lexer_lookahead.cpp index 694d7bd1e..51c509c96 100644 --- a/rcl/test/rcl/test_lexer_lookahead.cpp +++ b/rcl/test/rcl/test_lexer_lookahead.cpp @@ -108,6 +108,37 @@ TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_peek2) EXPECT_EQ(RCL_LEXEME_FORWARD_SLASH, lexeme2); } +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_eof) +{ + rcl_ret_t ret; + rcl_lexer_lookahead2_t buffer; + SCOPE_LOOKAHEAD2(buffer, ""); + + { + rcl_lexeme_t lexeme = RCL_LEXEME_NONE; + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret); + EXPECT_EQ(RCL_LEXEME_EOF, lexeme); + } + { + rcl_lexeme_t lexeme1 = RCL_LEXEME_NONE; + rcl_lexeme_t lexeme2 = RCL_LEXEME_NONE; + ret = rcl_lexer_lookahead2_peek2(&buffer, &lexeme1, &lexeme2); + EXPECT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); + EXPECT_EQ(RCL_LEXEME_EOF, lexeme1); + EXPECT_EQ(RCL_LEXEME_EOF, lexeme2); + } + // Accepting keeps the lexer at EOF + { + EXPECT_EQ(RCL_RET_OK, rcl_lexer_lookahead2_accept(&buffer, NULL, NULL)); + rcl_lexeme_t lexeme = RCL_LEXEME_NONE; + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); + EXPECT_EQ(RCL_RET_OK, ret); + EXPECT_EQ(RCL_LEXEME_EOF, lexeme); + } +} + + TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_accept) { rcl_ret_t ret; From 69f1f2a4d58b0044b1d2de7dd872290aca7b3fd3 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Thu, 12 Apr 2018 14:57:35 -0700 Subject: [PATCH 35/38] Test not zero-initialized init --- rcl/test/rcl/test_lexer_lookahead.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rcl/test/rcl/test_lexer_lookahead.cpp b/rcl/test/rcl/test_lexer_lookahead.cpp index 51c509c96..c26c8b4ab 100644 --- a/rcl/test/rcl/test_lexer_lookahead.cpp +++ b/rcl/test/rcl/test_lexer_lookahead.cpp @@ -66,6 +66,16 @@ TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_init_fini_ rcl_reset_error(); } +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_init_not_zero_initialized) +{ + rcl_lexer_lookahead2_t buffer; + const int not_zero = 1; + buffer.impl = reinterpret_cast(not_zero); + rcl_ret_t ret = rcl_lexer_lookahead2_init(&buffer, "foobar", rcl_get_default_allocator()); + EXPECT_EQ(RCL_RET_INVALID_ARGUMENT, ret); + rcl_reset_error(); +} + TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_peek) { rcl_ret_t ret; From b408d2b4ff3a71bb3ac7d5c7e5fe9916791405a1 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Thu, 12 Apr 2018 15:41:35 -0700 Subject: [PATCH 36/38] Test lexing remapping rules --- rcl/test/rcl/test_lexer_lookahead.cpp | 257 +++++++++++++++----------- 1 file changed, 152 insertions(+), 105 deletions(-) diff --git a/rcl/test/rcl/test_lexer_lookahead.cpp b/rcl/test/rcl/test_lexer_lookahead.cpp index c26c8b4ab..59e58ebed 100644 --- a/rcl/test/rcl/test_lexer_lookahead.cpp +++ b/rcl/test/rcl/test_lexer_lookahead.cpp @@ -212,116 +212,163 @@ TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_expect) EXPECT_EQ(RCL_RET_WRONG_LEXEME, ret) << rcl_get_error_string_safe(); } +#define EXPECT_LOOKAHEAD(expected_lexeme, expected_text, buffer) \ + do { \ + const char * lexeme_text; \ + size_t lexeme_text_length; \ + rcl_lexeme_t lexeme; \ + ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); \ + EXPECT_EQ(expected_lexeme, lexeme); \ + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); \ + ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); \ + ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); \ + EXPECT_STREQ(expected_text, std::string(lexeme_text, lexeme_text_length).c_str()); \ + } while (false) + TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_lex_long_string) { rcl_ret_t ret; rcl_lexer_lookahead2_t buffer; SCOPE_LOOKAHEAD2(buffer, ":\\1rostopic://\\2rosservice://~/\\8:=**:*foobar"); - const char * lexeme_text; - size_t lexeme_text_length; - rcl_lexeme_t lexeme; - - // : - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_COLON, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ(":", std::string(lexeme_text, lexeme_text_length).c_str()); - - // \1 - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_BR1, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("\\1", std::string(lexeme_text, lexeme_text_length).c_str()); - - // rostopic:// - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_URL_TOPIC, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("rostopic://", std::string(lexeme_text, lexeme_text_length).c_str()); - - // \2 - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_BR2, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("\\2", std::string(lexeme_text, lexeme_text_length).c_str()); - - // rosservice:// - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_URL_SERVICE, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("rosservice://", std::string(lexeme_text, lexeme_text_length).c_str()); - - // ~/ - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_TILDE_SLASH, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("~/", std::string(lexeme_text, lexeme_text_length).c_str()); - - // \8 - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_BR8, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("\\8", std::string(lexeme_text, lexeme_text_length).c_str()); - - // := - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_SEPARATOR, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ(":=", std::string(lexeme_text, lexeme_text_length).c_str()); - - // ** - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_WILD_MULTI, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("**", std::string(lexeme_text, lexeme_text_length).c_str()); - - // : - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_COLON, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ(":", std::string(lexeme_text, lexeme_text_length).c_str()); - - // * - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_WILD_ONE, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("*", std::string(lexeme_text, lexeme_text_length).c_str()); - // foobar - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_TOKEN, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("foobar", std::string(lexeme_text, lexeme_text_length).c_str()); + EXPECT_LOOKAHEAD(RCL_LEXEME_COLON, ":", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_BR1, "\\1", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_URL_TOPIC, "rostopic://", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_BR2, "\\2", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_URL_SERVICE, "rosservice://", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TILDE_SLASH, "~/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_BR8, "\\8", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_WILD_MULTI, "**", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_COLON, ":", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_WILD_ONE, "*", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foobar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); +} - // eof - ret = rcl_lexer_lookahead2_peek(&buffer, &lexeme); - EXPECT_EQ(RCL_LEXEME_EOF, lexeme); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - ret = rcl_lexer_lookahead2_accept(&buffer, &lexeme_text, &lexeme_text_length); - ASSERT_EQ(RCL_RET_OK, ret) << rcl_get_error_string_safe(); - EXPECT_STREQ("", std::string(lexeme_text, lexeme_text_length).c_str()); +TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_lex_remap_rules) +{ + rcl_ret_t ret; + rcl_lexer_lookahead2_t buffer; + { + SCOPE_LOOKAHEAD2(buffer, "foo:=bar"); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + SCOPE_LOOKAHEAD2(buffer, "/foo/bar:=fiz/buzz"); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "fiz", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "buzz", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Nodename prefix + SCOPE_LOOKAHEAD2(buffer, "nodename:~/foo:=foo"); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "nodename", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_COLON, ":", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TILDE_SLASH, "~/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Partial namespace replacement + SCOPE_LOOKAHEAD2(buffer, "/foo/**:=/fizz/\\1"); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_WILD_MULTI, "**", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "fizz", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_BR1, "\\1", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Full namespace replacement + SCOPE_LOOKAHEAD2(buffer, "/foo/bar/*:=/bar/foo/\\1"); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_WILD_ONE, "*", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_BR1, "\\1", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Change a base name + SCOPE_LOOKAHEAD2(buffer, "**/foo:=\\1/bar"); + EXPECT_LOOKAHEAD(RCL_LEXEME_WILD_MULTI, "**", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_BR1, "\\1", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Change namespace + SCOPE_LOOKAHEAD2(buffer, "__ns:=/new/namespace"); + EXPECT_LOOKAHEAD(RCL_LEXEME_NS, "__ns", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "new", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "namespace", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Change node name + SCOPE_LOOKAHEAD2(buffer, "__node:=left_camera_driver"); + EXPECT_LOOKAHEAD(RCL_LEXEME_NODE, "__node", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "left_camera_driver", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Topic only remap + SCOPE_LOOKAHEAD2(buffer, "rostopic://foo/bar:=bar/foo"); + EXPECT_LOOKAHEAD(RCL_LEXEME_URL_TOPIC, "rostopic://", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } + { + // Service only remap + SCOPE_LOOKAHEAD2(buffer, "rosservice:///foo/bar:=/bar/foo"); + EXPECT_LOOKAHEAD(RCL_LEXEME_URL_SERVICE, "rosservice://", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_SEPARATOR, ":=", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "bar", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_FORWARD_SLASH, "/", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_TOKEN, "foo", buffer); + EXPECT_LOOKAHEAD(RCL_LEXEME_EOF, "", buffer); + } } From 811e129a640cc19e5ae7ae89573140192f80052d Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 16 Apr 2018 07:42:28 -0700 Subject: [PATCH 37/38] Windows warning --- rcl/test/rcl/test_lexer_lookahead.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rcl/test/rcl/test_lexer_lookahead.cpp b/rcl/test/rcl/test_lexer_lookahead.cpp index 59e58ebed..6d169711d 100644 --- a/rcl/test/rcl/test_lexer_lookahead.cpp +++ b/rcl/test/rcl/test_lexer_lookahead.cpp @@ -70,7 +70,7 @@ TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_init_not_z { rcl_lexer_lookahead2_t buffer; const int not_zero = 1; - buffer.impl = reinterpret_cast(not_zero); + buffer.impl = reinterpret_cast(¬_zero); rcl_ret_t ret = rcl_lexer_lookahead2_init(&buffer, "foobar", rcl_get_default_allocator()); EXPECT_EQ(RCL_RET_INVALID_ARGUMENT, ret); rcl_reset_error(); From 3752a239fd7333e8d89149e415b419db72d78b7f Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Mon, 16 Apr 2018 09:26:18 -0700 Subject: [PATCH 38/38] Remove const to avoid discarding during cast --- rcl/test/rcl/test_lexer_lookahead.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rcl/test/rcl/test_lexer_lookahead.cpp b/rcl/test/rcl/test_lexer_lookahead.cpp index 6d169711d..54d0f285b 100644 --- a/rcl/test/rcl/test_lexer_lookahead.cpp +++ b/rcl/test/rcl/test_lexer_lookahead.cpp @@ -69,7 +69,7 @@ TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_init_fini_ TEST_F(CLASSNAME(TestLexerLookaheadFixture, RMW_IMPLEMENTATION), test_init_not_zero_initialized) { rcl_lexer_lookahead2_t buffer; - const int not_zero = 1; + int not_zero = 1; buffer.impl = reinterpret_cast(¬_zero); rcl_ret_t ret = rcl_lexer_lookahead2_init(&buffer, "foobar", rcl_get_default_allocator()); EXPECT_EQ(RCL_RET_INVALID_ARGUMENT, ret);