Skip to content

Commit

Permalink
Make the Parser independent from the global C-locale
Browse files Browse the repository at this point in the history
  • Loading branch information
vglavnyy committed Nov 11, 2018
1 parent f445c1e commit c5cbdfe
Show file tree
Hide file tree
Showing 12 changed files with 323 additions and 161 deletions.
19 changes: 13 additions & 6 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@ env:
global:
# Set at the root level as this is ignored when set under matrix.env.
- GCC_VERSION="4.9"
# Fail on first error if UBSAN or ASAN enabled for a target
- UBSAN_OPTIONS=halt_on_error=1
- ASAN_OPTIONS=halt_on_error=1
# Travis machines have 2 cores
- JOBS=2
- MAKEFLAGS="-j 2"

conan-linux: &conan-linux
os: linux
Expand Down Expand Up @@ -53,7 +59,7 @@ matrix:
# branch: master
- language: cpp
os:
- linux
- linux

compiler:
- gcc
Expand All @@ -79,8 +85,8 @@ matrix:
-DGRPC_INSTALL_PATH=$TRAVIS_BUILD_DIR/google/grpc/install
-DPROTOBUF_DOWNLOAD_PATH=$TRAVIS_BUILD_DIR/google/grpc/third_party/protobuf
-DFLATBUFFERS_CODE_SANITIZE=ON
- make
- LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib make test ARGS=-V
- cmake --build . -- -j${JOBS}
- LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib ctest --extra-verbose --output-on-failure
- bash .travis/check-generate-code.sh
- if [ "$CONAN" == "true" ] && [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo pip install conan && conan create . google/testing -s build_type=$BUILD_TYPE -tf conan/test_package; fi

Expand All @@ -91,6 +97,7 @@ matrix:
matrix:
- BUILD_TYPE=Debug
- BUILD_TYPE=Release

script:
- bash grpc/build_grpc.sh
- cmake .
Expand All @@ -99,10 +106,9 @@ matrix:
-DGRPC_INSTALL_PATH=$TRAVIS_BUILD_DIR/google/grpc/install
-DPROTOBUF_DOWNLOAD_PATH=$TRAVIS_BUILD_DIR/google/grpc/third_party/protobuf
-DFLATBUFFERS_CODE_SANITIZE=ON
- make
- ./flattests
- cmake --build . -- -j${JOBS}
- DYLD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib ctest --extra-verbose --output-on-failure
- bash .travis/check-generate-code.sh
- DYLD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib ./grpctest

- <<: *conan-linux
env: CONAN_GCC_VERSIONS=4.9 CONAN_DOCKER_IMAGE=lasote/conangcc49
Expand Down Expand Up @@ -146,6 +152,7 @@ matrix:
- extra-android-m2repository
compiler:
- gcc

before_install:
- git clone https://github.com/urho3d/android-ndk.git $HOME/android-ndk-root
- export ANDROID_NDK_HOME=$HOME/android-ndk-root
Expand Down
23 changes: 23 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
cmake_minimum_required(VERSION 2.8)
# generate compile_commands.json
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
include(CheckCXXSymbolExists)

project(FlatBuffers)

Expand Down Expand Up @@ -35,6 +36,16 @@ if(DEFINED FLATBUFFERS_MAX_PARSING_DEPTH)
message(STATUS "FLATBUFFERS_MAX_PARSING_DEPTH: ${FLATBUFFERS_MAX_PARSING_DEPTH}")
endif()

# Auto-detect locale-narrow 'strtod_l' function.
if(NOT DEFINED FLATBUFFERS_LOCALE_INDEPENDENT)
if(MSVC)
check_cxx_symbol_exists(_strtof_l stdlib.h FLATBUFFERS_LOCALE_INDEPENDENT)
else()
check_cxx_symbol_exists(strtof_l stdlib.h FLATBUFFERS_LOCALE_INDEPENDENT)
endif()
endif()
add_definitions(-DFLATBUFFERS_LOCALE_INDEPENDENT=$<BOOL:${FLATBUFFERS_LOCALE_INDEPENDENT}>)

set(FlatBuffers_Library_SRCS
include/flatbuffers/code_generators.h
include/flatbuffers/base.h
Expand Down Expand Up @@ -213,6 +224,7 @@ function(add_fsanitize_to_target _target _sanitizer)
target_link_libraries(${_target} PRIVATE
"-fsanitize${_sanitizer_flags}")
set_property(TARGET ${_target} PROPERTY POSITION_INDEPENDENT_CODE ON)
message(STATUS "Sanitizer ${_sanitizer_flags} added to ${_target}")
endif()
endfunction()

Expand Down Expand Up @@ -304,6 +316,17 @@ if(FLATBUFFERS_BUILD_TESTS)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/samples)
add_executable(flatsamplebinary ${FlatBuffers_Sample_Binary_SRCS})
add_executable(flatsampletext ${FlatBuffers_Sample_Text_SRCS})

if(DEFINED FLATBUFFERS_TEST_LOCALE)
# Enable test of locale independent code.
# -DFLATBUFFERS_TEST_LOCALE="" - test with default C-locale
# -DFLATBUFFERS_TEST_LOCALE="ru_RU.CP1251" - test with ru_RU.CP1251
# Locale was installed before (Ubuntu):>sudo locale-gen ru_RU.CP1251
target_compile_definitions(flattests PRIVATE
FLATBUFFERS_TEST_LOCALE=\"${FLATBUFFERS_TEST_LOCALE}\")
message(STATUS "FLATBUFFERS_TEST_LOCALE: \"${FLATBUFFERS_TEST_LOCALE}\"")
endif()

endif()

if(FLATBUFFERS_BUILD_GRPCTEST)
Expand Down
47 changes: 41 additions & 6 deletions docs/source/CppUsage.md
Original file line number Diff line number Diff line change
Expand Up @@ -499,11 +499,46 @@ To use scalars, simply wrap them in a struct.

## Depth limit of nested objects and stack-overflow control
The parser of Flatbuffers schema or json-files is kind of recursive parser.
To avoid stack-overflow problem the parser has a built-in limiter of recursion depth.
Number of nested declarations in a schema or number of nested json-objects is limited.
By default, this depth limit set to `64`.
It is possible to override this limit with `FLATBUFFERS_MAX_PARSING_DEPTH` definition.
This definition can be helpful for testing purposes or embedded applications.
For details see [build](@ref flatbuffers_guide_building) of CMake-based projects.
To avoid stack-overflow problem the parser has a built-in limiter of
recursion depth. Number of nested declarations in a schema or number of
nested json-objects is limited. By default, this depth limit set to `64`.
It is possible to override this limit with `FLATBUFFERS_MAX_PARSING_DEPTH`
definition. This definition can be helpful for testing purposes or embedded
applications. For details see [build](@ref flatbuffers_guide_building) of
CMake-based projects.

## Dependence from C-locale {#flatbuffers_locale_cpp}
The Flatbuffers [grammar](@ref flatbuffers grammar) uses ASCII
character set for identifiers, alphanumeric literals, reserved words.

Internal implementation of the Flatbuffers depends from functions which
depend from C-locale: `strtod()` or `strtof()`, for example.
The library expects the dot `.` symbol as the separator of an integer
part from the fractional part of a float number.
Another separator symbols (`,` for example) will break the compatibility
and may lead to an error while parsing a Flatbuffers schema or a json file.

The Standard C locale is a global resource, there is only one locale for
the entire application. Some modern compilers and platforms have
locale-independent or locale-narrow functions `strtof_l`, `strtod_l`,
`strtoll_l`, `strtoull_l` to resolve this dependency.
These functions use specified locale rather than the global or per-thread
locale instead. They are part of POSIX-2008 but not part of the C/C++
standard library, therefore, may be missing on some platforms.

The Flatbuffers library try to detect these functions at configuration and
compile time:
- `_MSC_VER >= 1900`: check MSVC2012 or higher for MSVC buid
- `_XOPEN_SOURCE>=700`: check POSIX-2008 for GCC/Clang build
- `check_cxx_symbol_exists(strtof_l stdlib.h)`: CMake check of `strtod_f`

After detection, the definition `FLATBUFFERS_LOCALE_INDEPENDENT` will be
set to `0` or `1`.

It is possible to test the compatibility of the Flatbuffers library with
a specific locale. Set the exact name of locale for the `flattests` target
using `FLATBUFFERS_TEST_LOCALE` definition, for example:
- `-D FLATBUFFERS_TEST_LOCALE=""` - default a system locale
- `-D FLATBUFFERS_TEST_LOCALE="ru_RU.CP1251"`

<br>
34 changes: 28 additions & 6 deletions include/flatbuffers/base.h
Original file line number Diff line number Diff line change
Expand Up @@ -195,15 +195,35 @@
#endif
#endif // !FLATBUFFERS_HAS_NEW_STRTOD

// Suppress sanitizer directives.
#ifndef FLATBUFFERS_LOCALE_INDEPENDENT
// Enable locale independent functions {strtof_l, strtod_l,strtoll_l, strtoull_l}.
// They are part of the POSIX-2008 but not part of the C/C++ standard.
// GCC/Clang have definition (_XOPEN_SOURCE>=700) if POSIX-2008.
#if ((defined(_MSC_VER) && _MSC_VER >= 1800) || \
(defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE>=700)))
#define FLATBUFFERS_LOCALE_INDEPENDENT 1
#else
#define FLATBUFFERS_LOCALE_INDEPENDENT 0
#endif
#endif // !FLATBUFFERS_LOCALE_INDEPENDENT

// Suppress Undefined Behavior Sanitizer (recoverable only). Usage:
// - __supress_ubsan__("undefined")
// - __supress_ubsan__("signed-integer-overflow")
#if defined(__clang__)
#define __no_sanitize_undefined__(reason) __attribute__((no_sanitize("undefined")))
#define __supress_ubsan__(type) __attribute__((no_sanitize(type)))
#elif defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 408)
#define __no_sanitize_undefined__(reason) __attribute__((no_sanitize_undefined))
#define __supress_ubsan__(type) __attribute__((no_sanitize_undefined))
#else
#define __no_sanitize_undefined__(reason)
#define __supress_ubsan__(type)
#endif

// This is constexpr function used for checking compile-time constants.
// Avoid `#pragma warning(disable: 4127) // C4127: expression is constant`.
template<typename T> FLATBUFFERS_CONSTEXPR inline bool IsConstTrue(T t) {
return !!t;
}

/// @endcond

/// @file
Expand Down Expand Up @@ -287,13 +307,15 @@ template<typename T> T EndianScalar(T t) {
}

template<typename T>
__no_sanitize_undefined__("C++ aliasing type rules, see std::bit_cast<>")
// UBSAN: C++ aliasing type rules, see std::bit_cast<> for details.
__supress_ubsan__("alignment")
T ReadScalar(const void *p) {
return EndianScalar(*reinterpret_cast<const T *>(p));
}

template<typename T>
__no_sanitize_undefined__("C++ aliasing type rules, see std::bit_cast<>")
// UBSAN: C++ aliasing type rules, see std::bit_cast<> for details.
__supress_ubsan__("alignment")
void WriteScalar(void *p, T t) {
*reinterpret_cast<T *>(p) = EndianScalar(t);
}
Expand Down
Loading

0 comments on commit c5cbdfe

Please sign in to comment.