diff --git a/CMakeLists.txt b/CMakeLists.txt index f29bb3187d..e71890327a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ include_directories(src/crypto) include_directories(src/crypto/ctaes) include_directories(src/crypto/external) include_directories(src/crypto/x16r) +include_directories(src/csv) include_directories(src/index) include_directories(src/interfaces) include_directories(src/leveldb/db) @@ -181,6 +182,11 @@ add_executable(veil src/crypto/sha256_sse41.cpp src/crypto/sha512.cpp src/crypto/sha512.h + src/csv/CSVread.cpp + src/csv/CSVwrite.cpp + src/csv/libcsv.c.cpp + src/csv/strerror.cpp + src/csv/strerror.hpp src/index/base.cpp src/index/base.h src/index/txindex.cpp diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index f147cee3b1..43087b2e68 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html # =========================================================================== # # SYNOPSIS @@ -33,21 +33,23 @@ # Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov # Copyright (c) 2015 Paul Norman # Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016, 2018 Krzesimir Nowak +# Copyright (c) 2019 Enji Cooper # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 4 +#serial 11 dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro dnl (serial version number 13). AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl - m4_if([$1], [11], [], - [$1], [14], [], - [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])], + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl m4_if([$2], [], [], [$2], [ext], [], @@ -57,26 +59,13 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], [$3], [optional], [ax_cxx_compile_cxx$1_required=false], [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) - m4_if([$4], [], [ax_cxx_compile_cxx$1_try_default=true], - [$4], [default], [ax_cxx_compile_cxx$1_try_default=true], - [$4], [nodefault], [ax_cxx_compile_cxx$1_try_default=false], - [m4_fatal([invalid fourth argument `$4' to AX_CXX_COMPILE_STDCXX])]) AC_LANG_PUSH([C++])dnl ac_success=no - m4_if([$4], [nodefault], [], [dnl - AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, - ax_cv_cxx_compile_cxx$1, - [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], - [ax_cv_cxx_compile_cxx$1=yes], - [ax_cv_cxx_compile_cxx$1=no])]) - if test x$ax_cv_cxx_compile_cxx$1 = xyes; then - ac_success=yes - fi]) - m4_if([$2], [noext], [], [dnl if test x$ac_success = xno; then - for switch in -std=gnu++$1 -std=gnu++0x; do + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, $cachevar, @@ -102,22 +91,27 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl dnl HP's aCC needs +std=c++11 according to: dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf dnl Cray's crayCC needs "-h std=c++11" - for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do - cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) - AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, - $cachevar, - [ac_save_CXX="$CXX" - CXX="$CXX $switch" - AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], - [eval $cachevar=yes], - [eval $cachevar=no]) - CXX="$ac_save_CXX"]) - if eval test x\$$cachevar = xyes; then - CXX="$CXX $switch" - if test -n "$CXXCPP" ; then - CXXCPP="$CXXCPP $switch" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break fi - ac_success=yes + done + if test x$ac_success = xyes; then break fi done @@ -154,6 +148,11 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 ) +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) dnl Tests for new features in C++11 @@ -191,11 +190,13 @@ namespace cxx11 struct Base { + virtual ~Base() {} virtual void f() {} }; struct Derived : public Base { + virtual ~Derived() override {} virtual void f() override {} }; @@ -524,7 +525,7 @@ namespace cxx14 } - namespace test_digit_seperators + namespace test_digit_separators { constexpr auto ten_million = 100'000'000; @@ -566,3 +567,385 @@ namespace cxx14 #endif // __cplusplus >= 201402L ]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include +#include +#include + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template + int multiply(Args... args) + { + return (args * ... * 1); + } + + template + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } + + namespace test_typename_in_template_template_parameter + { + + template typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/build-aux/m4/ax_pthread.m4 b/build-aux/m4/ax_pthread.m4 index 4c4051ea37..1598d077ff 100644 --- a/build-aux/m4/ax_pthread.m4 +++ b/build-aux/m4/ax_pthread.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_pthread.html +# https://www.gnu.org/software/autoconf-archive/ax_pthread.html # =========================================================================== # # SYNOPSIS @@ -55,6 +55,7 @@ # # Copyright (c) 2008 Steven G. Johnson # Copyright (c) 2011 Daniel Richard G. +# Copyright (c) 2019 Marc Stevens # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -67,7 +68,7 @@ # Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program. If not, see . +# with this program. If not, see . # # As a special exception, the respective Autoconf Macro's copyright owner # gives unlimited permission to copy, distribute and modify the configure @@ -82,7 +83,7 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 23 +#serial 27 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD]) AC_DEFUN([AX_PTHREAD], [ @@ -123,10 +124,12 @@ fi # (e.g. DEC) have both -lpthread and -lpthreads, where one of the # libraries is broken (non-POSIX). -# Create a list of thread flags to try. Items starting with a "-" are -# C compiler flags, and other items are library names, except for "none" -# which indicates that we try without any flags at all, and "pthread-config" -# which is a program returning the flags for the Pth emulation library. +# Create a list of thread flags to try. Items with a "," contain both +# C compiler flags (before ",") and linker flags (after ","). Other items +# starting with a "-" are C compiler flags, and remaining items are +# library names, except for "none" which indicates that we try without +# any flags at all, and "pthread-config" which is a program returning +# the flags for the Pth emulation library. ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" @@ -194,14 +197,47 @@ case $host_os in # that too in a future libc.) So we'll check first for the # standard Solaris way of linking pthreads (-mt -lpthread). - ax_pthread_flags="-mt,pthread pthread $ax_pthread_flags" + ax_pthread_flags="-mt,-lpthread pthread $ax_pthread_flags" ;; esac +# Are we compiling with Clang? + +AC_CACHE_CHECK([whether $CC is Clang], + [ax_cv_PTHREAD_CLANG], + [ax_cv_PTHREAD_CLANG=no + # Note that Autoconf sets GCC=yes for Clang as well as GCC + if test "x$GCC" = "xyes"; then + AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG], + [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */ +# if defined(__clang__) && defined(__llvm__) + AX_PTHREAD_CC_IS_CLANG +# endif + ], + [ax_cv_PTHREAD_CLANG=yes]) + fi + ]) +ax_pthread_clang="$ax_cv_PTHREAD_CLANG" + + # GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC) +# Note that for GCC and Clang -pthread generally implies -lpthread, +# except when -nostdlib is passed. +# This is problematic using libtool to build C++ shared libraries with pthread: +# [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=25460 +# [2] https://bugzilla.redhat.com/show_bug.cgi?id=661333 +# [3] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=468555 +# To solve this, first try -pthread together with -lpthread for GCC + AS_IF([test "x$GCC" = "xyes"], - [ax_pthread_flags="-pthread -pthreads $ax_pthread_flags"]) + [ax_pthread_flags="-pthread,-lpthread -pthread -pthreads $ax_pthread_flags"]) + +# Clang takes -pthread (never supported any other flag), but we'll try with -lpthread first + +AS_IF([test "x$ax_pthread_clang" = "xyes"], + [ax_pthread_flags="-pthread,-lpthread -pthread"]) + # The presence of a feature test macro requesting re-entrant function # definitions is, on some systems, a strong hint that pthreads support is @@ -224,25 +260,86 @@ AS_IF([test "x$ax_pthread_check_macro" = "x--"], [ax_pthread_check_cond=0], [ax_pthread_check_cond="!defined($ax_pthread_check_macro)"]) -# Are we compiling with Clang? -AC_CACHE_CHECK([whether $CC is Clang], - [ax_cv_PTHREAD_CLANG], - [ax_cv_PTHREAD_CLANG=no - # Note that Autoconf sets GCC=yes for Clang as well as GCC - if test "x$GCC" = "xyes"; then - AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG], - [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */ -# if defined(__clang__) && defined(__llvm__) - AX_PTHREAD_CC_IS_CLANG -# endif - ], - [ax_cv_PTHREAD_CLANG=yes]) - fi - ]) -ax_pthread_clang="$ax_cv_PTHREAD_CLANG" +if test "x$ax_pthread_ok" = "xno"; then +for ax_pthread_try_flag in $ax_pthread_flags; do + + case $ax_pthread_try_flag in + none) + AC_MSG_CHECKING([whether pthreads work without any flags]) + ;; + + *,*) + PTHREAD_CFLAGS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\1/"` + PTHREAD_LIBS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\2/"` + AC_MSG_CHECKING([whether pthreads work with "$PTHREAD_CFLAGS" and "$PTHREAD_LIBS"]) + ;; + + -*) + AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag]) + PTHREAD_CFLAGS="$ax_pthread_try_flag" + ;; + + pthread-config) + AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no]) + AS_IF([test "x$ax_pthread_config" = "xno"], [continue]) + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; + + *) + AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag]) + PTHREAD_LIBS="-l$ax_pthread_try_flag" + ;; + esac + + ax_pthread_save_CFLAGS="$CFLAGS" + ax_pthread_save_LIBS="$LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + + # Check for various functions. We must include pthread.h, + # since some functions may be macros. (On the Sequent, we + # need a special flag -Kthread to make this header compile.) + # We check for pthread_join because it is in -lpthread on IRIX + # while pthread_create is in libc. We check for pthread_attr_init + # due to DEC craziness with -lpthreads. We check for + # pthread_cleanup_push because it is one of the few pthread + # functions on Solaris that doesn't have a non-functional libc stub. + # We try pthread_create on general principles. + + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include +# if $ax_pthread_check_cond +# error "$ax_pthread_check_macro must be defined" +# endif + static void *some_global = NULL; + static void routine(void *a) + { + /* To avoid any unused-parameter or + unused-but-set-parameter warning. */ + some_global = a; + } + static void *start_routine(void *a) { return a; }], + [pthread_t th; pthread_attr_t attr; + pthread_create(&th, 0, start_routine, 0); + pthread_join(th, 0); + pthread_attr_init(&attr); + pthread_cleanup_push(routine, 0); + pthread_cleanup_pop(0) /* ; */])], + [ax_pthread_ok=yes], + []) + + CFLAGS="$ax_pthread_save_CFLAGS" + LIBS="$ax_pthread_save_LIBS" + + AC_MSG_RESULT([$ax_pthread_ok]) + AS_IF([test "x$ax_pthread_ok" = "xyes"], [break]) + + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" +done +fi -ax_pthread_clang_warning=no # Clang needs special handling, because older versions handle the -pthread # option in a rather... idiosyncratic way @@ -261,11 +358,6 @@ if test "x$ax_pthread_clang" = "xyes"; then # -pthread does define _REENTRANT, and while the Darwin headers # ignore this macro, third-party headers might not.) - PTHREAD_CFLAGS="-pthread" - PTHREAD_LIBS= - - ax_pthread_ok=yes - # However, older versions of Clang make a point of warning the user # that, in an invocation where only linking and no compilation is # taking place, the -pthread option has no effect ("argument unused @@ -320,78 +412,7 @@ if test "x$ax_pthread_clang" = "xyes"; then fi # $ax_pthread_clang = yes -if test "x$ax_pthread_ok" = "xno"; then -for ax_pthread_try_flag in $ax_pthread_flags; do - - case $ax_pthread_try_flag in - none) - AC_MSG_CHECKING([whether pthreads work without any flags]) - ;; - - -mt,pthread) - AC_MSG_CHECKING([whether pthreads work with -mt -lpthread]) - PTHREAD_CFLAGS="-mt" - PTHREAD_LIBS="-lpthread" - ;; - - -*) - AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag]) - PTHREAD_CFLAGS="$ax_pthread_try_flag" - ;; - - pthread-config) - AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no]) - AS_IF([test "x$ax_pthread_config" = "xno"], [continue]) - PTHREAD_CFLAGS="`pthread-config --cflags`" - PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" - ;; - *) - AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag]) - PTHREAD_LIBS="-l$ax_pthread_try_flag" - ;; - esac - - ax_pthread_save_CFLAGS="$CFLAGS" - ax_pthread_save_LIBS="$LIBS" - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LIBS="$PTHREAD_LIBS $LIBS" - - # Check for various functions. We must include pthread.h, - # since some functions may be macros. (On the Sequent, we - # need a special flag -Kthread to make this header compile.) - # We check for pthread_join because it is in -lpthread on IRIX - # while pthread_create is in libc. We check for pthread_attr_init - # due to DEC craziness with -lpthreads. We check for - # pthread_cleanup_push because it is one of the few pthread - # functions on Solaris that doesn't have a non-functional libc stub. - # We try pthread_create on general principles. - - AC_LINK_IFELSE([AC_LANG_PROGRAM([#include -# if $ax_pthread_check_cond -# error "$ax_pthread_check_macro must be defined" -# endif - static void routine(void *a) { a = 0; } - static void *start_routine(void *a) { return a; }], - [pthread_t th; pthread_attr_t attr; - pthread_create(&th, 0, start_routine, 0); - pthread_join(th, 0); - pthread_attr_init(&attr); - pthread_cleanup_push(routine, 0); - pthread_cleanup_pop(0) /* ; */])], - [ax_pthread_ok=yes], - []) - - CFLAGS="$ax_pthread_save_CFLAGS" - LIBS="$ax_pthread_save_LIBS" - - AC_MSG_RESULT([$ax_pthread_ok]) - AS_IF([test "x$ax_pthread_ok" = "xyes"], [break]) - - PTHREAD_LIBS="" - PTHREAD_CFLAGS="" -done -fi # Various other checks: if test "x$ax_pthread_ok" = "xyes"; then @@ -438,7 +459,8 @@ if test "x$ax_pthread_ok" = "xyes"; then AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT], [ax_cv_PTHREAD_PRIO_INHERIT], [AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[int i = PTHREAD_PRIO_INHERIT;]])], + [[int i = PTHREAD_PRIO_INHERIT; + return i;]])], [ax_cv_PTHREAD_PRIO_INHERIT=yes], [ax_cv_PTHREAD_PRIO_INHERIT=no]) ]) diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 05f2207d23..a976086eef 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -79,25 +79,12 @@ AC_DEFUN([BITCOIN_QT_INIT],[ AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path) ]) -dnl Find the appropriate version of Qt libraries and includes. -dnl Inputs: $1: Whether or not pkg-config should be used. yes|no. Default: yes. -dnl Inputs: $2: If $1 is "yes" and --with-gui=auto, which qt version should be -dnl tried first. -dnl Outputs: See _BITCOIN_QT_FIND_LIBS_* +dnl Find Qt libraries and includes. +dnl Outputs: See _BITCOIN_QT_FIND_LIBS dnl Outputs: Sets variables for all qt-related tools. dnl Outputs: bitcoin_enable_qt, bitcoin_enable_qt_dbus, bitcoin_enable_qt_test AC_DEFUN([BITCOIN_QT_CONFIGURE],[ - use_pkgconfig=$1 - - if test "x$use_pkgconfig" = x; then - use_pkgconfig=yes - fi - - if test "x$use_pkgconfig" = xyes; then - BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG]) - else - BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG]) - fi + BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS]) dnl This is ugly and complicated. Yuck. Works as follows: dnl For Qt5, we can check a header to find out whether Qt is build @@ -152,7 +139,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ CXXFLAGS=$TEMP_CXXFLAGS ]) - if test "x$use_pkgconfig$qt_bin_path" = xyes; then + if test "x$qt_bin_path" = x; then qt_bin_path="`$PKG_CONFIG --variable=host_bins Qt5Core 2>/dev/null`" fi @@ -226,7 +213,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ dnl enable qt support - AC_MSG_CHECKING(whether to build ]AC_PACKAGE_NAME[ GUI) + AC_MSG_CHECKING([whether to build ]AC_PACKAGE_NAME[ GUI]) BITCOIN_QT_CHECK([ bitcoin_enable_qt=yes bitcoin_enable_qt_test=yes @@ -264,57 +251,16 @@ dnl All macros below are internal and should _not_ be used from the main dnl configure.ac. dnl ---- -dnl Internal. Check if the included version of Qt is Qt5. -dnl Requires: INCLUDES must be populated as necessary. -dnl Output: bitcoin_cv_qt5=yes|no -AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ - AC_CACHE_CHECK(for Qt 5, bitcoin_cv_qt5,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #ifndef QT_VERSION - # include - #endif - ]], - [[ - #if QT_VERSION < 0x050000 || QT_VERSION_MAJOR < 5 - choke - #endif - ]])], - [bitcoin_cv_qt5=yes], - [bitcoin_cv_qt5=no]) -])]) - -dnl Internal. Check if the included version of Qt is greater than Qt58. -dnl Requires: INCLUDES must be populated as necessary. -dnl Output: bitcoin_cv_qt5=yes|no -AC_DEFUN([_BITCOIN_QT_CHECK_QT58],[ - AC_CACHE_CHECK(for > Qt 5.7, bitcoin_cv_qt58,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #ifndef QT_VERSION - # include - #endif - ]], - [[ - #if QT_VERSION_MINOR < 8 - choke - #endif - ]])], - [bitcoin_cv_qt58=yes], - [bitcoin_cv_qt58=no]) -])]) - dnl Internal. Check if the linked version of Qt was built as static libs. dnl Requires: Qt5. dnl Requires: INCLUDES and LIBS must be populated as necessary. dnl Output: bitcoin_cv_static_qt=yes|no -dnl Output: Defines QT_STATICPLUGIN if plugins are static. AC_DEFUN([_BITCOIN_QT_IS_STATIC],[ AC_CACHE_CHECK(for static Qt, bitcoin_cv_static_qt,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include - #ifndef QT_VERSION OR QT_VERSION_STR + #ifndef QT_VERSION # include #endif ]], @@ -326,9 +272,6 @@ AC_DEFUN([_BITCOIN_QT_IS_STATIC],[ [bitcoin_cv_static_qt=yes], [bitcoin_cv_static_qt=no]) ]) - if test "x$bitcoin_cv_static_qt" = xyes; then - AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol for static Qt plugins]) - fi ]) dnl Internal. Check if the link-requirements for static plugins are met. @@ -359,73 +302,32 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ if test -d "$qt_plugin_path/accessible"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible" fi - if test "x$use_pkgconfig" = xyes; then - : dnl - m4_ifdef([PKG_CHECK_MODULES],[ - if test x$bitcoin_cv_qt58 = xno; then - PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"]) - else - PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) - fi - if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([X11XCB], [x11-xcb], [QT_LIBS="$X11XCB_LIBS $QT_LIBS"]) - if ${PKG_CONFIG} --exists "Qt5Core >= 5.5" 2>/dev/null; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) - fi - elif test "x$TARGET_OS" = xdarwin; then - PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) - fi - ]) - else - if test "x$TARGET_OS" = xwindows; then - AC_CACHE_CHECK(for Qt >= 5.6, bitcoin_cv_need_platformsupport,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #ifndef QT_VERSION - # include - #endif - ]], - [[ - #if QT_VERSION < 0x050600 || QT_VERSION_MINOR < 6 - choke - #endif - ]])], - [bitcoin_cv_need_platformsupport=yes], - [bitcoin_cv_need_platformsupport=no]) - ]) - if test "x$bitcoin_cv_need_platformsupport" = xyes; then - if test x$bitcoin_cv_qt58 = xno; then - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}PlatformSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXPlatformSupport not found))) - else - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FontDatabaseSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFontDatabaseSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}EventDispatcherSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXEventDispatcherSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}ThemeSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXThemeSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FbSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFbSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}DeviceDiscoverySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXDeviceDiscoverySupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}AccessibilitySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXAccessibilitySupport not found))) - QT_LIBS="$QT_LIBS -lversion -ldwmapi -luxtheme" - fi - fi - fi - fi - fi + m4_ifdef([PKG_CHECK_MODULES],[ + if test x$bitcoin_cv_qt58 = xno; then + PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"]) + else + PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) + fi + if test "x$TARGET_OS" = xlinux; then + PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) + fi + ]) + fi ]) dnl Internal. Find Qt libraries using pkg-config. -dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to check -dnl first. -dnl Inputs: $1: If bitcoin_qt_want_version is "auto", check for this version -dnl first. dnl Outputs: All necessary QT_* variables are set. dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. -AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG],[ +AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ m4_ifdef([PKG_CHECK_MODULES],[ QT_LIB_PREFIX=Qt5 qt5_modules="Qt5Core Qt5Gui Qt5Network Qt5Widgets" @@ -446,85 +348,3 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG],[ ]) true; dnl ]) - -dnl Internal. Find Qt libraries without using pkg-config. Version is deduced -dnl from the discovered headers. -dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to use. -dnl If "auto", the version will be discovered by _BITCOIN_QT_CHECK_QT5. -dnl Outputs: All necessary QT_* variables are set. -dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. -AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[ - TEMP_CPPFLAGS="$CPPFLAGS" - TEMP_CXXFLAGS="$CXXFLAGS" - CXXFLAGS="$PIC_FLAGS $CXXFLAGS" - TEMP_LIBS="$LIBS" - BITCOIN_QT_CHECK([ - if test "x$qt_include_path" != x; then - QT_INCLUDES="-I$qt_include_path -I$qt_include_path/QtCore -I$qt_include_path/QtGui -I$qt_include_path/QtWidgets -I$qt_include_path/QtNetwork -I$qt_include_path/QtTest -I$qt_include_path/QtDBus" - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - fi - ]) - - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QtPlugin],,BITCOIN_QT_FAIL(QtCore headers missing))]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QApplication],, BITCOIN_QT_FAIL(QtGui headers missing))]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QLocalSocket],, BITCOIN_QT_FAIL(QtNetwork headers missing))]) - - BITCOIN_QT_CHECK([ - if test "x$bitcoin_qt_want_version" = xauto; then - _BITCOIN_QT_CHECK_QT5 - _BITCOIN_QT_CHECK_QT58 - fi - QT_LIB_PREFIX=Qt5 - ]) - - BITCOIN_QT_CHECK([ - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="$LIBS -L$qt_lib_path" - fi - - if test "x$TARGET_OS" = xwindows; then - AC_CHECK_LIB([imm32], [main],, BITCOIN_QT_FAIL(libimm32 not found)) - fi - ]) - - BITCOIN_QT_CHECK(AC_CHECK_LIB([z] ,[main],,AC_MSG_WARN([zlib not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([jpeg_create_decompress] ,[qtjpeg jpeg],,AC_MSG_WARN([libjpeg not found. Assuming qt has it built-in]))) - if test x$bitcoin_cv_qt58 = xno; then - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre16_exec], [qtpcre pcre16],,AC_MSG_WARN([libpcre16 not found. Assuming qt has it built-in]))) - else - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtlibpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre2_match_16], [qtpcre2 libqtpcre2],,AC_MSG_WARN([libqtpcre2 not found. Assuming qt has it built-in]))) - fi - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([hb_ot_tags_from_script] ,[qtharfbuzzng qtharfbuzz harfbuzz],,AC_MSG_WARN([libharfbuzz not found. Assuming qt has it built-in or support is disabled]))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Core] ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Core not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Gui] ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Gui not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Network],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Network not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Widgets],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Widgets not found))) - QT_LIBS="$LIBS" - LIBS="$TEMP_LIBS" - - BITCOIN_QT_CHECK([ - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="-L$qt_lib_path" - fi - AC_CHECK_LIB([${QT_LIB_PREFIX}Test], [main],, have_qt_test=no) - AC_CHECK_HEADER([QTest],, have_qt_test=no) - QT_TEST_LIBS="$LIBS" - if test "x$use_dbus" != xno; then - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="-L$qt_lib_path" - fi - AC_CHECK_LIB([${QT_LIB_PREFIX}DBus], [main],, have_qt_dbus=no) - AC_CHECK_HEADER([QtDBus],, have_qt_dbus=no) - QT_DBUS_LIBS="$LIBS" - fi - ]) - CPPFLAGS="$TEMP_CPPFLAGS" - CXXFLAGS="$TEMP_CXXFLAGS" - LIBS="$TEMP_LIBS" -]) - diff --git a/configure.ac b/configure.ac index 0ea2fbb474..a0446506e9 100644 --- a/configure.ac +++ b/configure.ac @@ -14,6 +14,12 @@ AC_CONFIG_HEADERS([src/config/veil-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) +m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) +PKG_PROG_PKG_CONFIG +if test "x$PKG_CONFIG" = x; then + AC_MSG_ERROR([pkg-config not found]) +fi + dnl Add config links for asm includes for RandomX AC_CONFIG_LINKS([src/crypto/randomx/asm/program_epilogue_linux.inc:src/crypto/randomx/asm/program_epilogue_linux.inc]) AC_CONFIG_LINKS([src/crypto/randomx/asm/program_epilogue_store.inc:src/crypto/randomx/asm/program_epilogue_store.inc]) @@ -77,8 +83,20 @@ case $host in lt_cv_deplibs_check_method="pass_all" ;; esac -dnl Require C++11 compiler (no GNU extensions) -AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) + +AC_ARG_ENABLE([c++17], + [AS_HELP_STRING([--enable-c++17], + [enable compilation in c++17 mode (disabled by default)])], + [use_cxx17=$enableval], + [use_cxx17=no]) + +dnl Require C++11 or C++17 compiler (no GNU extensions) +if test "x$use_cxx17" = xyes; then + AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) +else + AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) +fi + dnl Check if -latomic is required for CHECK_ATOMIC @@ -435,13 +453,8 @@ AC_ARG_WITH([daemon], [build_bitcoind=$withval], [build_bitcoind=yes]) -use_pkgconfig=yes case $host in *mingw*) - - #pkgconfig does more harm than good with MinGW - use_pkgconfig=no - TARGET_OS=windows AC_CHECK_LIB([mingwthrd], [main],, AC_MSG_ERROR(libmingwthrd missing)) AC_CHECK_LIB([kernel32], [main],, AC_MSG_ERROR(libkernel32 missing)) @@ -477,7 +490,7 @@ case $host in AC_MSG_ERROR("windres not found") fi - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB" + CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" LEVELDB_TARGET_FLAGS="-DOS_WINDOWS" if test "x$CXXFLAGS_overridden" = "xno"; then CXXFLAGS="$CXXFLAGS -w" @@ -500,6 +513,8 @@ case $host in archive_cmds_CXX="\$CC -shared \$libobjs \$deplibs \$compiler_flags -static -o \$output_objdir/\$soname \${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker \$lib" postdeps_CXX= + dnl We require Windows 7 (NT 6.1) or later + AX_CHECK_LINK_FLAG([[-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1]],[LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"],,[[$LDFLAG_WERROR]]) ;; *darwin*) TARGET_OS=darwin @@ -530,16 +545,17 @@ case $host in dnl the user (--without-wallet or --without-gui for example). openssl_prefix=`$BREW --prefix openssl 2>/dev/null` - bdb_prefix=`$BREW --prefix berkeley-db4 2>/dev/null` - qt5_prefix=`$BREW --prefix qt5 2>/dev/null` if test x$openssl_prefix != x; then PKG_CONFIG_PATH="$openssl_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH fi - if test x$bdb_prefix != x; then - CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include" - LIBS="$LIBS -L$bdb_prefix/lib" + if test "x$use_bdb" != xno && $BREW list --versions berkeley-db4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then + bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) + dnl This must precede the call to BITCOIN_FIND_BDB48 below. + BDB_CFLAGS="-I$bdb_prefix/include" + BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi + qt5_prefix=$($BREW --prefix qt5 2>/dev/null) if test x$qt5_prefix != x; then PKG_CONFIG_PATH="$qt5_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH @@ -568,7 +584,7 @@ case $host in fi AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"]) - CPPFLAGS="$CPPFLAGS -DMAC_OSX" + CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; *android*) @@ -609,16 +625,6 @@ case $host in ;; esac -if test x$use_pkgconfig = xyes; then - m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR(PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh.)]) - m4_ifdef([PKG_PROG_PKG_CONFIG], [ - PKG_PROG_PKG_CONFIG - if test x"$PKG_CONFIG" = "x"; then - AC_MSG_ERROR(pkg-config not found.) - fi - ]) -fi - if test x$use_extended_functional_tests != xno; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi @@ -1090,86 +1096,53 @@ fi fi -if test x$use_pkgconfig = xyes; then - : dnl - m4_ifdef( - [PKG_CHECK_MODULES], - [ - PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) - PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) - BITCOIN_QT_CHECK([PKG_CHECK_MODULES([PROTOBUF], [protobuf], [have_protobuf=yes], [BITCOIN_QT_FAIL(libprotobuf not found)])]) - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) - fi - if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then - PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) - if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) - fi - fi - - if test "x$use_zmq" = "xyes"; then - PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) - use_zmq=no]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi - ] - ) -else - AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing)) - AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing)) + PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) + PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) + BITCOIN_QT_CHECK([PKG_CHECK_MODULES([PROTOBUF], [protobuf], [have_protobuf=yes], [BITCOIN_QT_FAIL(libprotobuf not found)])]) - AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),) - AC_CHECK_LIB([ssl], [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing)) +dnl libevent check - if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then - AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) - AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) - if test x$TARGET_OS != xwindows; then - AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) - fi +if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then + PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) + if test x$TARGET_OS != xwindows; then + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) fi +fi - if test "x$use_zmq" = "xyes"; then - AC_CHECK_HEADER([zmq.h], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_MSG_WARN([zmq.h not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - AC_CHECK_LIB([zmq],[zmq_ctx_shutdown],ZMQ_LIBS=-lzmq, - [AC_MSG_WARN([libzmq >= 4.0 not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi +dnl QR Code encoding library check - if test "x$use_zmq" = "xyes"; then - dnl Assume libzmq was built for static linking - case $host in - *mingw*) - ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" - ;; - esac - fi +if test "x$use_qr" != xno; then + BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) +fi - BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) - fi +#save_CXXFLAGS="${CXXFLAGS}" +#CXXFLAGS="${CXXFLAGS} ${CRYPTO_CFLAGS} ${SSL_CFLAGS}" +#AC_CHECK_DECLS([EVP_MD_CTX_new],,,[AC_INCLUDES_DEFAULT +##include +#]) +#CXXFLAGS="${save_CXXFLAGS}" + +dnl ZMQ check + +if test "x$use_zmq" = xyes; then + PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], + AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), + [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) + AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + use_zmq=no]) +else + AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) fi -save_CXXFLAGS="${CXXFLAGS}" -CXXFLAGS="${CXXFLAGS} ${CRYPTO_CFLAGS} ${SSL_CFLAGS}" -AC_CHECK_DECLS([EVP_MD_CTX_new],,,[AC_INCLUDES_DEFAULT -#include -]) -CXXFLAGS="${save_CXXFLAGS}" +if test "x$use_zmq" = xyes; then + dnl Assume libzmq was built for static linking + case $host in + *mingw*) + ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" + ;; + esac + BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) +fi dnl univalue check @@ -1178,43 +1151,24 @@ need_bundled_univalue=yes if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononono; then need_bundled_univalue=no else - -if test x$system_univalue != xno ; then - found_univalue=no - if test x$use_pkgconfig = xyes; then - : #NOP - m4_ifdef( - [PKG_CHECK_MODULES], - [ - PKG_CHECK_MODULES([UNIVALUE],[libunivalue >= 1.0.4],[found_univalue=yes],[true]) - ] - ) - else - AC_CHECK_HEADER([univalue.h],[ - AC_CHECK_LIB([univalue], [main],[ - UNIVALUE_LIBS=-lunivalue - found_univalue=yes - ],[true]) - ],[true]) + if test x$system_univalue != xno; then + PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) + if test x$found_univalue = xyes; then + system_univalue=yes + need_bundled_univalue=no + elif test x$system_univalue = xyes; then + AC_MSG_ERROR([univalue not found]) + else + system_univalue=no + fi fi - if test x$found_univalue = xyes ; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes ; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no + if test x$need_bundled_univalue = xyes; then + UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' + UNIVALUE_LIBS='univalue/libunivalue.la' fi fi -if test x$need_bundled_univalue = xyes ; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' -fi - -fi - AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) @@ -1563,10 +1517,10 @@ echo " target os = $TARGET_OS" echo " build os = $BUILD_OS" echo echo " CC = $CC" -echo " CFLAGS = $CFLAGS" +echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_CFLAGS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 21749a2e1d..4d612f8dd1 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -14,7 +14,7 @@ READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') -NONFATAL = {} # checks which are non-fatal for now but only generate a warning +OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') def check_ELF_PIE(executable): ''' @@ -115,25 +115,18 @@ def check_ELF_Canary(executable): ok = True return ok -def get_PE_dll_characteristics(executable): - ''' - Get PE DllCharacteristics bits. - Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' - and bits is the DllCharacteristics value. - ''' +def get_PE_dll_characteristics(executable) -> int: + '''Get PE DllCharacteristics bits''' p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') - arch = '' bits = 0 for line in stdout.splitlines(): tokens = line.split() - if len(tokens)>=2 and tokens[0] == 'architecture:': - arch = tokens[1].rstrip(',') if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': bits = int(tokens[1],16) - return (arch,bits) + return bits IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 @@ -141,25 +134,19 @@ def get_PE_dll_characteristics(executable): def check_PE_DYNAMIC_BASE(executable): '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - (arch,bits) = get_PE_dll_characteristics(executable) - reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE - return (bits & reqbits) == reqbits + bits = get_PE_dll_characteristics(executable) + return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE -# On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE -# to have secure ASLR. +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. def check_PE_HIGH_ENTROPY_VA(executable): '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - (arch,bits) = get_PE_dll_characteristics(executable) - if arch == 'i386:x86-64': - reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA - else: # Unnecessary on 32-bit - assert(arch == 'i386') - reqbits = 0 - return (bits & reqbits) == reqbits + bits = get_PE_dll_characteristics(executable) + return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA def check_PE_NX(executable): '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - (arch,bits) = get_PE_dll_characteristics(executable) + bits = get_PE_dll_characteristics(executable) return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT CHECKS = { @@ -196,18 +183,12 @@ def identify_executable(executable): continue failed = [] - warning = [] for (name, func) in CHECKS[etype]: if not func(filename): - if name in NONFATAL: - warning.append(name) - else: - failed.append(name) + failed.append(name) if failed: print('%s: failed %s' % (filename, ' '.join(failed))) retval = 1 - if warning: - print('%s: warning %s' % (filename, ' '.join(warning))) except IOError: print('%s: cannot open' % filename) retval = 1 diff --git a/depends/config.site.in b/depends/config.site.in index 8444dc26f2..5fd5ed42e7 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -38,15 +38,6 @@ if test x@host_os@ = xdarwin; then PORT=no fi -if test x@host_os@ = xmingw32; then - if test -z $with_qt_incdir; then - with_qt_incdir=$depends_prefix/include - fi - if test -z $with_qt_libdir; then - with_qt_libdir=$depends_prefix/lib - fi -fi - PATH=$depends_prefix/native/bin:$PATH PKG_CONFIG="`which pkg-config` --static" diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index a1c943d60b..5b215cc55e 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -2,6 +2,9 @@ OSX_MIN_VERSION=10.10 OSX_SDK_VERSION=10.11 OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk LD64_VERSION=253.9 + +# When cross-compiling for Darwin using Clang, -mlinker-version must be passed to +# ensure that modern linker features are enabled. darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 5f622f8e6e..895cf7601d 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,15 +3,21 @@ $(package)_version=2.1.8-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz $(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d +$(package)_patches=0001-fix-windows-getaddrinfo.patch define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/0001-fix-windows-getaddrinfo.patch && \ ./autogen.sh endef +# When building for Windows, we set _WIN32_WINNT to target the same Windows +# version as we do in configure. Due to quirks in libevents build system, this +# is also required to enable support for ipv6. See #19375. define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic + $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 endef define $(package)_config_cmds diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index d5fd1f39ab..41fb9e4c74 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,20 +1,22 @@ package=zeromq -$(package)_version=4.2.3 +$(package)_version=4.3.1 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=8f1e2b2aade4dbfde98d82366d61baef2f62e812530160d2e6d0a5bb24e40bc0 -$(package)_patches=0001-fix-build-with-older-mingw64.patch 0002-disable-pthread_set_name_np.patch +$(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb +$(package)_patches=remove_libstd_link.patch define $(package)_set_vars - $(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf --disable-Werror + $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf + $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci + $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking + $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic $(package)_cxxflags=-std=c++11 endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/0001-fix-build-with-older-mingw64.patch && \ - patch -p1 < $($(package)_patch_dir)/0002-disable-pthread_set_name_np.patch && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config + patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config endef define $(package)_config_cmds @@ -30,6 +32,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \ - rm -rf bin share + rm -rf bin share lib/*.la endef diff --git a/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch new file mode 100644 index 0000000000..a98cd90bd5 --- /dev/null +++ b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch @@ -0,0 +1,15 @@ +diff -ur libevent-2.1.8-stable.orig/configure.ac libevent-2.1.8-stable/configure.ac +--- libevent-2.1.8-stable.orig/configure.ac 2017-01-29 17:51:00.000000000 +0000 ++++ libevent-2.1.8-stable/configure.ac 2020-03-07 01:11:16.311335005 +0000 +@@ -389,6 +389,10 @@ + #ifdef HAVE_NETDB_H + #include + #endif ++#ifdef _WIN32 ++#include ++#include ++#endif + ]], + [[ + getaddrinfo; +Only in libevent-2.1.8-stable: configure.ac~ diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch index 34302a9f2d..8c722ffb46 100644 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ b/depends/patches/qt/fix_qt_pkgconfig.patch @@ -1,11 +1,23 @@ --- old/qtbase/mkspecs/features/qt_module.prf +++ new/qtbase/mkspecs/features/qt_module.prf -@@ -245,7 +245,7 @@ +@@ -264,7 +264,7 @@ load(qt_targets) # this builds on top of qt_common -!internal_module:!lib_bundle:if(unix|mingw) { -+unix|mingw { ++if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { CONFIG += create_pc QMAKE_PKGCONFIG_DESTDIR = pkgconfig host_build: \ +@@ -274,9 +274,9 @@ + QMAKE_PKGCONFIG_INCDIR = $$[QT_INSTALL_HEADERS/raw] + QMAKE_PKGCONFIG_CFLAGS = -I${includedir}/$$MODULE_INCNAME + QMAKE_PKGCONFIG_NAME = $$replace(TARGET, ^Qt, "Qt$$QT_MAJOR_VERSION ") +- QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION) ++ QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION)$$qtPlatformTargetSuffix() + for(i, MODULE_DEPENDS): \ +- QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0)) ++ QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0))$$qtPlatformTargetSuffix() + isEmpty(QMAKE_PKGCONFIG_DESCRIPTION): \ + QMAKE_PKGCONFIG_DESCRIPTION = $$replace(TARGET, ^Qt, "Qt ") module + pclib_replace.match = $$lib_replace.match diff --git a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch b/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch deleted file mode 100644 index a6c508fb8a..0000000000 --- a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 1a159c128c69a42d90819375c06a39994f3fbfc1 Mon Sep 17 00:00:00 2001 -From: Cory Fields -Date: Tue, 28 Nov 2017 20:33:25 -0500 -Subject: [PATCH] fix build with older mingw64 - ---- - src/windows.hpp | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/src/windows.hpp b/src/windows.hpp -index 99e889d..e69038e 100644 ---- a/src/windows.hpp -+++ b/src/windows.hpp -@@ -55,6 +55,13 @@ - #include - #include - #include -+ -+#if defined __MINGW64_VERSION_MAJOR && __MINGW64_VERSION_MAJOR < 4 -+// Workaround for mingw-w64 < v4.0 which did not include ws2ipdef.h in iphlpapi.h. -+// Fixed in mingw-w64 by 9bd8fe9148924840d315b4c915dd099955ea89d1. -+#include -+#include -+#endif - #include - - #if !defined __MINGW32__ --- -2.7.4 - diff --git a/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch b/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch deleted file mode 100644 index d220b54f3e..0000000000 --- a/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 6e6b47d5ab381c3df3b30bb0b0a6cf210dfb1eba Mon Sep 17 00:00:00 2001 -From: Cory Fields -Date: Mon, 5 Mar 2018 14:22:05 -0500 -Subject: [PATCH] disable pthread_set_name_np - -pthread_set_name_np adds a Glibc requirement on >= 2.12. ---- - src/thread.cpp | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/src/thread.cpp b/src/thread.cpp -index 4fc59c3e..c3fdfd46 100644 ---- a/src/thread.cpp -+++ b/src/thread.cpp -@@ -220,7 +220,7 @@ void zmq::thread_t::setThreadName(const char *name_) - */ - if (!name_) - return; -- -+#if 0 - #if defined(ZMQ_HAVE_PTHREAD_SETNAME_1) - int rc = pthread_setname_np(name_); - if(rc) return; -@@ -233,6 +233,8 @@ void zmq::thread_t::setThreadName(const char *name_) - #elif defined(ZMQ_HAVE_PTHREAD_SET_NAME) - pthread_set_name_np(descriptor, name_); - #endif -+#endif -+ return; - } - - #endif --- -2.11.1 - diff --git a/doc/build-osx.md b/doc/build-osx.md index 10b1d86993..314fcd4cc4 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -1,101 +1,116 @@ -macOS Build Instructions and Notes -==================================== +# macOS Build Instructions and Notes + The commands in this guide should be executed in a Terminal application. -The built-in one is located in `/Applications/Utilities/Terminal.app`. +The built-in one is located in +``` +/Applications/Utilities/Terminal.app +``` -Preparation ------------ +## Preparation Install the macOS command line tools: -`xcode-select --install` +```shell +xcode-select --install +``` When the popup appears, click `Install`. Then install [Homebrew](https://brew.sh). -Dependencies ----------------------- - - brew install automake berkeley-db4 libtool boost miniupnpc openssl pkg-config protobuf python qt libevent qrencode gmp +## Dependencies +```shell + brew install automake libtool boost miniupnpc openssl pkg-config protobuf python qt libevent qrencode gmp +``` +If you run into issues, check [Homebrew's troubleshooting page](https://docs.brew.sh/Troubleshooting). See [dependencies.md](dependencies.md) for a complete overview. -If you want to build the disk image with `make deploy` (.dmg / optional), you need RSVG +If you want to build the disk image with `make deploy` (.dmg / optional), you need RSVG: +```shell +brew install librsvg +``` - brew install librsvg - -Clone the Repo --------------- - -1. Clone the Veil source code and cd into `veil` - git clone https://github.com/Veil-Project/veil.git +#### Berkeley DB -Berkeley DB ------------ It is recommended to use Berkeley DB 4.8. If you have to build it yourself, -you can use [the installation script included in contrib/](/contrib/install_db4.sh) -like so +you can use [this](/contrib/install_db4.sh) script to install it +like so: ```shell -./contrib/install_db4.sh . +CFLAGS="-Wno-error=implicit-function-declaration" ./contrib/install_db4.sh . ``` from the root of the repository. **Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). -Build Veil Core ------------------------- - -1. Cd into `veil` - - cd veil +## Build Veil Core +1. Clone the Veil source code and cd into `veil` + ```shell + git clone https://github.com/Veil-Project/veil.git + + cd veil + ``` 2. Build Veil: Configure and build the headless Veil binaries as well as the GUI (if Qt is found). You can disable the GUI build by passing `--without-gui` to configure. - - ./autogen.sh - ./configure - make + ```shell + ./autogen.sh + ./configure + make + ``` 3. It is recommended to build and run the unit tests: + ```shell + make check + ``` + +4. You can also create a `.dmg` that contains the `.app` bundle (optional): + ```shell + make deploy + ``` + +## Disable-wallet mode +When the intention is to run only a P2P node without a wallet, Veil may be +compiled in disable-wallet mode with: +```shell +./configure --disable-wallet +``` - make check - -4. You can also create a .dmg that contains the .app bundle (optional): - - make deploy +In this case there is no dependency on [*Berkeley DB*](#berkeley-db) and [*SQLite*](#sqlite). -Running -------- +Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. +## Running Veil is now available at `./src/veild` Before running, it's recommended that you create an RPC configuration file. - + ```shell echo -e "rpcuser=veilrpc\nrpcpassword=$(xxd -l 16 -p /dev/urandom)" > "/Users/${USER}/Library/Application Support/Veil/veil.conf" chmod 600 "/Users/${USER}/Library/Application Support/Veil/veil.conf" + ``` -The first time you run veild, it will start downloading the blockchain. This process could take several hours. +The first time you run veild, it will start downloading the blockchain. This process could +take many hours, or even days on slower than average systems. You can monitor the download process by looking at the debug.log file: - + ```shell tail -f $HOME/Library/Application\ Support/Veil/debug.log + ``` -Other commands: -------- - +## Other commands: +```shell ./src/veild -daemon # Starts the veil daemon. ./src/veil-cli --help # Outputs a list of command-line options. ./src/veil-cli help # Outputs a list of RPC commands when the daemon is running. +``` -Notes ------ +## Notes * Tested on OS X 10.10 Yosemite through macOS 10.13 High Sierra on 64-bit Intel processors only. diff --git a/doc/build-windows.md b/doc/build-windows.md index c711e1c3b7..b46cc6ee42 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -89,13 +89,22 @@ Acquire the source in the usual way: Once the source code is ready the build steps are below. +Additional WSL Note: WSL support for [launching Win32 applications](https://docs.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) +results in `Autoconf` configure scripts being able to execute Windows Portable Executable files. This can cause +unexpected behaviour during the build, such as Win32 error dialogs for missing libraries. The recommended approach +is to temporarily disable WSL support for Win32 applications. + +Build using: + PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var + sudo bash -c "echo 0 > /proc/sys/fs/binfmt_misc/status" # Disable WSL support for Win32 applications. cd depends make HOST=x86_64-w64-mingw32 cd .. ./autogen.sh # not required when building from tarball CONFIG_SITE=$PWD/depends/x86_64-w64-mingw32/share/config.site ./configure --prefix=/ make + sudo bash -c "echo 1 > /proc/sys/fs/binfmt_misc/status" # Enable WSL support for Win32 applications. ## Building for 32-bit Windows diff --git a/src/Makefile.am b/src/Makefile.am index ee3642acae..9018a09274 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,10 +4,11 @@ DIST_SUBDIRS = secp256k1 univalue -AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps +PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = if EMBEDDED_UNIVALUE @@ -115,6 +116,8 @@ BITCOIN_CORE_H = \ consensus/tx_verify.h \ core_io.h \ core_memusage.h \ + csv/csv.h \ + csv/CSV.hpp \ cuckoocache.h \ fs.h \ httprpc.h \ @@ -141,6 +144,7 @@ BITCOIN_CORE_H = \ netbase.h \ netmessagemaker.h \ noui.h \ + optional.h \ outputtype.h \ policy/feerate.h \ policy/fees.h \ @@ -689,6 +693,11 @@ libbitcoin_util_a_SOURCES = \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ + csv/CSVread.cpp \ + csv/CSVwrite.cpp \ + csv/libcsv.c \ + csv/strerror.cpp \ + csv/strerror.hpp \ fs.cpp \ interfaces/handler.cpp \ interfaces/node.cpp \ @@ -725,7 +734,7 @@ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h veild_SOURCES = veild.cpp veild_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) veild_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -veild_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +veild_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS veild_SOURCES += veild-res.rc @@ -752,7 +761,7 @@ veild_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_ veil_cli_SOURCES = veil-cli.cpp veil_cli_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) veil_cli_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -veil_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +veil_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS veil_cli_SOURCES += veil-cli-res.rc @@ -771,7 +780,7 @@ veil_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) veil_tx_SOURCES = veil-tx.cpp veil_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) veil_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -veil_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +veil_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS veil_tx_SOURCES += veil-tx-res.rc @@ -820,6 +829,7 @@ CLEANFILES += consensus/*.gcda consensus/*.gcno CLEANFILES += crypto/*.gcda crypto/*.gcno CLEANFILES += crypto/randomx/*.gcda crypto/randomx/*.gcno CLEANFILES += crypto/randomx/blake2/*.gcda crypto/randomx/blake2/*.gcno +CLEANFILES += csv/*.gcda csv/*.gcno CLEANFILES += policy/*.gcda policy/*.gcno CLEANFILES += primitives/*.gcda primitives/*.gcno CLEANFILES += script/*.gcda script/*.gcno diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 024970ee15..006095cfda 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -57,7 +57,7 @@ bench_bench_veil_SOURCES += bench/coin_selection.cpp endif bench_bench_veil_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -bench_bench_veil_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bench_bench_veil_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 3a5f803506..1a95699af4 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -750,7 +750,7 @@ endif qt_veil_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_ZEROCOIN) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -qt_veil_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +qt_veil_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) qt_veil_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX #locale/foo.ts -> locale/foo.qm diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index f1b434948f..4a851e6406 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -66,7 +66,7 @@ qt_test_test_veil_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOI $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -qt_test_test_veil_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +qt_test_test_veil_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) qt_test_test_veil_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) CLEAN_BITCOIN_QT_TEST = $(TEST_QT_MOC_CPP) qt/test/*.gcda qt/test/*.gcno diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 77ee5942e0..3c5b822b48 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -127,7 +127,7 @@ test_test_veil_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMM test_test_veil_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_veil_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) -test_test_veil_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static +test_test_veil_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -static if ENABLE_ZMQ test_test_veil_LDADD += $(ZMQ_LIBS) @@ -153,7 +153,7 @@ test_test_veil_fuzzy_LDADD = \ $(LIBBITCOIN_CRYPTO_SHANI) \ $(LIBSECP256K1) -test_test_veil_fuzzy_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_test_veil_fuzzy_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) $(PTHREAD_FLAGS) # nodist_test_test_veil_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 9677d72998..afb2297d07 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ template bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data) { // Generate random temporary filename - unsigned short randv = 0; + uint16_t randv = 0; GetRandBytes((unsigned char*)&randv, sizeof(randv)); std::string tmpfn = strprintf("%s.%04x", prefix, randv); diff --git a/src/compat.h b/src/compat.h index 31cae2b7f1..664e1bb29b 100644 --- a/src/compat.h +++ b/src/compat.h @@ -21,13 +21,6 @@ #endif #ifdef WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN 1 -#endif #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/core_write.cpp b/src/core_write.cpp index 50e63e1aef..48cff4dff3 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -134,7 +134,7 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serializeFlags); ssTx << tx; - return HexStr(ssTx.begin(), ssTx.end()); + return HexStr(ssTx); } void AddRangeproof(const std::vector &vRangeproof, UniValue &entry) @@ -222,7 +222,7 @@ void OutputToJSON(uint256 &txid, int i, void ScriptToUniv(const CScript& script, UniValue& out, bool include_address) { out.pushKV("asm", ScriptToAsmStr(script)); - out.pushKV("hex", HexStr(script.begin(), script.end())); + out.pushKV("hex", HexStr(script)); std::vector> solns; txnouttype type; @@ -244,7 +244,7 @@ void ScriptPubKeyToUniv(const CScript& scriptPubKey, out.pushKV("asm", ScriptToAsmStr(scriptPubKey)); if (fIncludeHex) - out.pushKV("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + out.pushKV("hex", HexStr(scriptPubKey)); if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired)) { out.pushKV("type", GetTxnOutputType(type)); @@ -276,7 +276,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, const std::vecto const CTxIn& txin = tx.vin[i]; UniValue in(UniValue::VOBJ); if (tx.IsCoinBase()) - in.pushKV("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + in.pushKV("coinbase", HexStr(txin.scriptSig)); if (txin.IsAnonInput()) { in.pushKV("type", "anon"); uint32_t nSigInputs, nSigRingSize; @@ -313,7 +313,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, const std::vecto in.pushKV("vout", (int64_t)txin.prevout.n); UniValue o(UniValue::VOBJ); o.pushKV("asm", ScriptToAsmStr(txin.scriptSig, true)); - o.pushKV("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + o.pushKV("hex", HexStr(txin.scriptSig)); in.pushKV("scriptSig", o); if (!tx.vin[i].scriptWitness.IsNull()) { UniValue txinwitness(UniValue::VARR); diff --git a/src/csv/CSV.hpp b/src/csv/CSV.hpp new file mode 100644 index 0000000000..bfc144ca22 --- /dev/null +++ b/src/csv/CSV.hpp @@ -0,0 +1,729 @@ +/* +Copyright (C) 2014 Jay Satiro +All rights reserved. + +This file is part of CSV/jay::util. + +https://github.com/jay/CSV + +jay::util is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +jay::util is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with jay::util. If not, see . +*/ + +/** Usage and design: + +Classes to read/write CSV records from/to a stream. + +If a bool in these classes is described as "set" without indicating true or false, it means true. + + +class CSVread +- A class to read comma separated values from a stream. + +class CSVwrite +- A class to write comma separated values to a stream. + + +These classes use libcsv --a powerful well written C library-- to parse the CSV records. Libcsv will +parse binary CSV data. If you pass in a filename it is opened in binary mode unless you specify the +flag 'text_mode'. Text mode does translations which may be undesirable for your data. If you pass in +an existing istream the 'text_mode' flag is not used; you are responsible for making sure it's in +the right mode for your data. + +Here is an example for CSVread: + +Car car; +jay::util::CSVread csv( "filename" ); +if( csv.error ) { initialization failed, handle it } +while( csv.ReadRecord() ) +{ +// assuming your field size is fixed at 4, check it first: +if( csv.fields.size() != 4 ) {handle it. error, continue, etc} +car.make = csv.fields[ 0 ]; +car.model = csv.fields[ 1 ]; +car.year = atoi( csv.fields[ 2 ] ); +car.description = csv.fields[ 3 ]; +Process(car); +} + +For more examples refer to ..\Example\Example.sln +*/ + +#ifndef JAY_UTIL_CSV_HPP_ +#define JAY_UTIL_CSV_HPP_ + +#include + +#include +#include +#include +#include + + +struct csv_parser; + +namespace jay { +namespace util { + + + +class CSVread +{ +public: + // flags for Open()/Associate() + enum Flags + { + none = 0, + + /* Skip the UTF-8 BOM check. + + 'has_utf8_bom' will always be false if the check is skipped. + + There is no UTF-8 processing done on the stream either way; the check is just to identify + and then ignore the BOM if present. You may pass this flag if you are certain your CSV + file/istream does not have a UTF-8 BOM. + */ + skip_utf8_bom_check = 1 << 0, + + + /* Process empty records. + + An empty record is a record that has nothing or only whitespace before its terminator. By + default empty records are ignored and not processed as records. + + A record that consists of just a field separator --eg a comma `,`-- is *not* considered an + empty record by this parser, it is parsed as two empty fields. + + Regarding its default behavior of ignoring empty records libcsv says: + "This behavior is meant to accommodate files using only either a linefeed or a carriage + return as a record separator to be parsed properly while at the same time being able to + parse files with rows terminated by multiple characters from resulting in blank rows after + each actual row of data (for example, processing a text CSV file created that was created on + a Windows machine on a Unix machine)." + + I implemented the processing of empty records by ignoring carriage returns as a record + terminator in that case. If you are certain all records end in LF or CRLF but never just CR, + and you want to process empty records then you may pass this flag. An exception to the must + have LF or CRLF rule is the end record, since it's allowed to not have any terminator. + + All empty records must be terminated to be processed. That means an end record cannot be + unterminated AND empty, which in that unique case means the stream ends in trailing + whitespace that is ignored regardless. + + Keep in mind if using this flag that because each empty record is now considered a record + it will have its own record number; therefore the record number corresponding to each + record could be different depending on whether or not this flag is passed to Open(). + + Internally CSV_REPALL_NL is set for the csv parser when this flag is set. + */ + process_empty_records = 1 << 1, + + + /* Strict mode. + + By default libcsv's parser is forgiving and will parse a field even if it does not strictly + adhere to the most common CSV usage. More details can be found in libcsv's manual and + http://www.creativyst.com/Doc/Articles/CSV/CSV01.htm + + You may pass this flag to error when parsing any format that's outside of common usage. + + An end record missing a newline terminator is considered common usage unfortunately, and + strict mode will not error. To handle that condition eval 'end_record_not_terminated'. + + Internally CSV_STRICT and CSV_STRICT_FINI are set for the csv parser when this flag is set. + */ + strict_mode = 1 << 2, + + + /* Error on NULL byte in field. + + If a field contains null bytes those bytes are considered part of the field by default. A + field is exposed as std::string, and it may be undesirable to allow NULL bytes in a string. + */ + error_on_null_in_field = 1 << 3, + + + /* Open file in text mode instead of binary mode. + + Libcsv handles files in binary mode and by default files are opened in binary mode by this + class. This should be fine for most purposes since libcsv autodetects CR,LF,CRLF as record + terminators. However if you have records with multiline data that is not binary then you + may prefer the translation. In that case you may associate an already existing istream that + has that translation enabled or pass in this flag with your file. + + An error will occur if you use this flag when associating an existing istream. This is + subject to change, should I decide to roll my own translation someday for binary streams. + */ + text_mode = 1 << 4 + }; + + + /* Constructor + + The constructor calls Init(). + The constructor also calls Open() if a filename is specified. + The constructor also calls Associate() if an istream is specified. + + In any case check 'error' to determine whether or not construction succeeded. + + [in] 'filename' : A file to open for input. + [in] 'stream' : An istream already opened for input. + [in][opt] 'flags' : Refer to CSVread::Flags. The default is no flags are set. + */ + CSVread(); + CSVread( std::string filename, Flags flags = none ); + CSVread( std::istream *stream, Flags flags = none ); + + // If a file was opened by this class it's automatically closed when the class destructs. + ~CSVread(); + + + /* CSVread::Reset() + - Reset to an empty state. + + Resets most variables, calls ResetParser() and ResetCache(). + + The delimiter is not reset. To reset the delimiter call SetDelimiter( ',' ). + The size of the buffer is not reset. To reset the size of the buffer call ResizeBuffer(). + + If the file/istream is open/associated it's clear()'d, then its position is reset. If the + file/istream is not seekable do not call this function. The return code is undefined if the + stream is not seekable. Instead call Close() which will close/dissociate first before it calls + this function. + + It shouldn't be necessary to call this function, typically. If there was an error and you wanted + to reset without associating/dissociating your istream, or opening/closing your file, you would + call this function. + + [in] 'partial_reset' : A partial reset does everything described above but does not the reset + 'fields', 'record_num', 'end_record_num', 'end_record_not_terminated'. Typically there + should be no reason to set this true. The default is false (a full reset). + [ret][failure] (false) : 'error' and 'error_msg' have been set. + [ret][success] (true) + */ + bool Reset( bool partial_reset = false ); + + + // Closes file if open or dissociates the existing istream, and then calls Reset(). + // If this returns false the file/istream has been closed/dissociated but Reset() failed. + bool Close(); + + // This is the same as Close(). It may make your code easier to understand to call this when you + // are dissociating a stream not created by the class because it's not closed just dissociated. + bool Dissociate() { return Close(); }; + + + /* CSVread::Open(), CSVread::Associate() + - Open a file or associate an existing istream. + + If there is already an open file or associated istream this function fails. + + If this function fails for any reason you must call Close() to reset before trying again. + + When the class destructs if there is a file that was opened by this function it is automatically + closed, but you may call Close() before then. + + If the stream starts with a UTF-8 BOM those bytes are ignored unless flag + CSVread::skip_utf8_bom_check is passed. There is no UTF-8 processing done on the stream + either way; the check is just to identify and then ignore the BOM if present. + + Once the stream has been opened you may call ReadRecord() to retrieve each record. + + The beginning of the file/istream current position must be the beginning of a record with an + optional UTF8 BOM. Keep in mind the first record read is 'record_num' 1. + + If the file/istream is not seekable there are some limitations in ReadRecord(). + + [in] 'filename' : A file to open for input. + [in] 'stream' : An istream already opened for input. + [in][opt] 'flags' : Refer to CSVread::Flags. The default is no flags are set. + [ret][failure] (false) : 'error' and 'error_msg' are set. + [ret][success] (true) : 'has_utf8_bom' may be set. + */ + bool Open( std::string filename, Flags flags = none ); + bool Associate( std::istream *stream, Flags flags = none ); + + + /* CSVread::GetDelimiter(), CSVread::SetDelimiter() + - Get or set the delimiter character to be used when parsing the stream. + + The default delimiter is a comma and does not need to be set. + + The delimiter is also known as the field separator character. Typically SetDelimiter() would be + used to set the delimiter to a semi-colon instead of the default comma. + + The delimiter character must not conflict with any whitespace, terminator or qualifier (default: + double quote) character. No error checking is done because the advanced features of libcsv allow + all of those to be changed and therefore in rare circumstances one of those characters could be + technically acceptable as the delimiter. + + The delimiter is persistent and will survive resets. It doesn't need to be set on each open. + */ + unsigned char GetDelimiter(); + void SetDelimiter( unsigned char delim ); + + + /* CSVread::ReadRecord() + - Read and parse a record. + + If the requested record number 'requested_record_num' is specified (ie not the default 0) and it + is less than the current record number 'record_num' then this function has to parse the stream + from the beginning to get to the requested record and is inefficient. In addition if the stream + is not seekable this function will fail in that case. + + The end record is not known until this function attempts to read past it, the same way the EOF + of a stream is not known until you attempt to read past its end. Since records are parsed and + cached in advance the end record may or may not be known when you successfully request a record, + including what may be the end record. For instructions on testing for the end record refer to + the comment block above the declaration for 'end_record_num'. + + 'eof', 'end_record_num' or 'end_record_not_terminated' set on success is not an indication that + the record returned is the end record. You should keep calling ReadRecord(), until it fails, to + read any remaining records in the cache. + + [in][opt] 'requested_record_num' : The record number to read. The default is the next record. + [ret][failure] (false) : 'error' and 'error_msg' are set; + 'eof', 'end_record_num' and 'end_record_not_terminated' may also be set. + [ret][success] (true) : 'record_num' and 'fields' are set; + 'eof', 'end_record_num' and 'end_record_not_terminated' may also be set. + */ + bool ReadRecord( const uintmax_t requested_record_num = 0 ); + + + /* Change the size of the buffer, in bytes. + + The buffer exists for the life of the object. It has a default size of 4096 bytes and is used to + hold data read from the stream. + + The cache list may allocate the same amount of memory as the size of the buffer at any time. + + Libcsv has its own buffer that is unaffected by this setting. + + [ret][failure] (false) : The buffer could not be resized and has retained its current size. + 'error' and 'error_msg' are set. + [ret][success] (true) + */ + bool ResizeBuffer( const std::streamsize bytes ); + + + // The size of _buffer. Default 4096. Call ResizeBuffer() to change the size. + const std::streamsize &buffer_size; // = _buffer_size + + /* Stream EOF. + + EOF is just an indication that the end of the stream was reached. As noted in ReadRecord(), EOF + does not indicate the state of the records. If you want to determine whether the current record + is the end record refer to 'end_record_num'. + */ + const bool &eof; // = _eof + + // Error. Functions will not succeed when this is true. Call Reset() or Close() or Dissociate(). + const bool &error; // = _error + + // Contains an error message when 'error' or '_pending_error'. + // If 'error' you can read this string before calling Reset() or Close() or Dissociate(). + const std::string &error_msg; // = _error_msg + + // File/istream has a UTF-8 BOM + const bool &has_utf8_bom; // = _has_utf8_bom + + // The record number of the current record. + // The first CSV record is record number 1. + const uintmax_t &record_num; // = _record_num + + + /* The record number of the end record. + + This is set if 'eof' is true and all records were parsed successfully. + + The value of this variable is undefined when 'eof' is false. + + In order to parse the end record, every record before it will have been parsed successfully, + even if those records were ignored (eg you skipped them). Therefore when this is set all + records were parsed successfully. + + Testing for the end record should come after a call to ReadRecord() fails, not before. There is + no guarantee on ReadRecord() success that the record read is known to be the end record even if + it is actually the end record. + + // process all records including the end record + while( csv_read.ReadRecord() ) + { + } + + // determine if the end record was the last successfully read + if( obj.eof && ( obj.record_num == obj.end_record_num ) ) + { + } + + The example above assumes you are reading consecutively. Assuming the if statement is true, + since ReadRecord() does not set 'fields' or 'record_num' on failure they would still represent + the end record in that case. + */ + const uintmax_t &end_record_num; // = _end_record_num + + + /* The end record was not terminated. + + This is true if 'end_record_num' is set and the end record was not terminated. + + Common usage of the CSV format does not require the end record to be terminated using a newline + sequence, defined as LF, CRLF or just CR (unless flag 'process_empty_records' was specified). + Despite this it's strongly recommended to always terminate in a newline and correct the + condition where it's missing. If you have this condition and something appends records to your + CSV file it likely will not check earlier records, so what was previously the end record in your + file and the first appended record would become one corrupted record. + */ + const bool &end_record_not_terminated; // = _end_record_not_terminated + + + // The current record parsed into fields (columns), eg fields[0], fields[1], etc. + // Depending on your CSV data the number of fields may or may not differ between records. + // You can check fields.size() after each ReadRecord() to confirm the number is what's expected. + const std::vector &fields; // = _fields + + + /* The libcsv parse object. + + You may use this to set more advanced behavior. + + To access this object you must include libcsv's "csv.h" in your code. + + Do not call these functions on parse_obj: + csv_init(), csv_set_opts(), csv_parse(), csv_fini(), csv_free(), csv_get_delim(), + csv_set_delim(). + + To get/set the delimiter call Get/SetDelimiter() instead. + + If 'error' parse_obj is not guaranteed != NULL or a good state; don't call any libcsv function. + */ + struct ::csv_parser *parse_obj; + + +private: + CSVread( const CSVread & ); + CSVread & operator=( const CSVread & ); + + // Call this to reset parse_obj. + bool ResetParser(); + + // A file stream if one was opened by this class. + std::ifstream _file; + + // The stream the records are read from. + // This points to the user specified istream or _file. + std::istream *_input_ptr; + + // A list of the records (each as a vector containing the fields) most recently parsed. + // The back element is always used for the pending record. The size of the cache should never + // be less than 1. + // To clear this list call ResetCache(), which starts a new list with an empty back element. + std::list> _cache; + + // Call this to reset _cache. + // The new list starts with one empty vector as the pending record. + void ResetCache(); + + // Whether or not an error is pending. Sometimes this is set instead of '_error' if there are + // completed records still in the cache when an error occurs. When the completed records are + // emptied if '_error_pending' then '_error' is set. See ReadRecord() definition. + bool _error_pending; + + // The flags passed to Open()/Associate(). + Flags _flags; + + // The buffer used to hold data read from the stream. + char *_buffer; + + // The field separator character. + unsigned char _delimiter; // = , + + // For a description of any of these refer to their public const references. + std::streamsize _buffer_size; + bool _eof; + bool _error; + std::string _error_msg; + bool _has_utf8_bom; + uintmax_t _record_num; + uintmax_t _end_record_num; + bool _end_record_not_terminated; + std::vector _fields; + + // Initialization to be called from the constructor only. + bool Init(); +}; + +inline CSVread::Flags operator | (CSVread::Flags a, CSVread::Flags b) +{ + return CSVread::Flags( ( (int)a ) | ( (int)b ) ); +} +inline CSVread::Flags & operator |= (CSVread::Flags &a, CSVread::Flags b) +{ + return (CSVread::Flags &)( ( (int &)a ) |= ( (int)b ) ); +} + + + +class CSVwrite +{ +public: + // flags for Open()/Associate() + enum Flags + { + none = 0, + + /* Truncate file. + + Any contents that existed in the file before it is open are discarded. + + An error will occur if you use this flag when associating an existing ostream. This is + subject to change, should I decide to handle discarding data in an existing ostream. + */ + truncate = 1 << 0, + + /* Open file in text mode instead of binary mode. + + Records are written in binary mode by default. That means that characters are + untranslated. + + You may prefer the translation done by text mode. In that case you may associate an already + existing stream that has that translation enabled or pass in this flag with your file. + + An error will occur if you use this flag when associating an existing ostream. This is + subject to change, should I decide to roll my own translation someday for binary streams. + */ + text_mode = 1 << 1, + + /* Process empty records. + + By default empty records (no fields) are ignored and not processed as records. + + Empty records are completely empty. If you choose to process empty records only a + terminator is written for the empty record. + + This flag changes the behavior of WriteRecord() but not WriteTerminator(). If you call + WriteTerminator() without first writing a field or record then you have written an empty + record, regardless of whether or not this flag is used. + */ + process_empty_records = 1 << 2 + }; + + + /* Constructor + + The constructor calls Init(). + The constructor also calls Open() if a filename is specified. + The constructor also calls Associate() if an ostream is specified. + + In any case check 'error' to determine whether or not construction succeeded. + + [in] 'filename' : A file to open for output. + [in] 'stream' : An ostream already opened for output. + [in][opt] 'flags' : Refer to CSVwrite::Flags. The default is no flags are set. + */ + CSVwrite(); + CSVwrite( std::string filename, Flags flags = none ); + CSVwrite( std::ostream *stream, Flags flags = none ); + + // If a file was opened by this class it's automatically closed when the class destructs. + ~CSVwrite(); + + + // Closes file if open or dissociates the existing ostream, and then calls Reset(). + // If this returns false the file/ostream has been closed/dissociated but Reset() failed. + bool Close(); + + // This is the same as Close(). It may make your code easier to understand to call this when you + // are dissociating a stream not created by the class because it's not closed just dissociated. + bool Dissociate() { return Close(); }; + + + /* CSVwrite::Open(), CSVwrite::Associate() + - Open a file or associate an existing ostream. + + A file is opened in append mode by default. To truncate the file instead pass the flag + CSVwrite::truncate. + + If there is already an open file or associated istream this function fails. + + If this function fails for any reason you must call Close() to reset before trying again. + + When the class destructs if there is a file that was opened by this function it is automatically + closed, but you may call Close() before then. + + If you are writing UTF-8 data and are writing from the beginning of the file/ostream and need + the UTF-8 BOM call WriteUTF8BOM() after opening. + + Once the stream has been opened you may call WriteRecord() to write each record or WriteField() + to write fields one at a time. + + The current position of the file/ostream must be the beginning of a record with an optional UTF8 + BOM if it's the beginning of the file/ostream. If you are appending it's important that the end + record already in the file/ostream is terminated, otherwise there will be corruption when + appending the first record. For more on this refer to the comment block above the declaration + for CSVread::end_record_not_terminated. + + [in] 'filename' : A file to open for output. + [in] 'stream' : An ostream already opened for output. + [in][opt] 'flags' : Refer to CSVwrite::Flags. The default is no flags are set. + [ret][failure] (false) : 'error' and 'error_msg' are set. + [ret][success] (true) + */ + bool Open( std::string filename, Flags flags = none ); + bool Associate( std::ostream *stream, Flags flags = none ); + + + /* CSVwrite::WriteUTF8BOM() + - Write a UTF-8 BOM. + + If at the beginning of a file/ostream you can call this to write a UTF-8 BOM, if necessary. + + [ret][failure] (false) : 'error' and 'error_msg' are set. + [ret][success] (true) + */ + bool WriteUTF8BOM(); + + + /* CSVwrite::WriteTerminator() + - Write a record terminator. + + If you are writing fields individually and you do not use the terminate option, or if you are + writing a record and you do not use the terminate option, you can terminate the record by + calling this function. + + [ret][failure] (false) : 'error' and 'error_msg' are set. + [ret][success] (true) + */ + bool WriteTerminator(); + + + /* CSVwrite::WriteField() + - Write a field. + + When writing the last field in the record set 'terminate' true or call WriteTerminator(). + + Every field is automatically qualified with quotes; you do not need to add your own qualifiers. + + [in] 'field' : The field. + [in][opt] 'terminate' : Terminate the record. The default is false. + [ret][failure] (false) : 'error' and 'error_msg' are set. + [ret][success] (true) + */ + bool WriteField( const std::string &field, const bool terminate = false ); + + + /* CSVwrite::WriteRecord() + - Write a record. + + All records should be terminated. If you set 'terminate' false make sure to call + WriteTerminator(). + + If this function is called and the previous record written by either this function or + WriteField() wasn't terminated then it is terminated before processing 'fields'. + + If 'fields' has a size of 0 (no fields- empty record) nothing is written unless + flag CSVwrite::process_empty_records was specified, in which case a terminator is written. + + Every field is automatically qualified with quotes; you do not need to add your own qualifiers. + + [in] 'fields' : The record. + [in][opt] 'terminate' : Terminate the record. The default is true. + [ret][failure] (false) : 'error' and 'error_msg' are set. + [ret][success] (true) + */ + bool WriteRecord( const std::vector &fields, const bool terminate = true ); + + + /* Change the size of the buffer, in bytes. + + The buffer exists for the life of the object. It has a default size of 4096 bytes and is used to + hold data before it's written to the stream. + + The buffer will hold at most a single field before writing it, therefore it's not as useful to + change the buffer size for this as it is for CSVread's buffer, unless your fields typically + are greater than 4096 bytes. + + [ret][failure] (false) : The buffer could not be resized and has retained its current size. + 'error' and 'error_msg' are set. + [ret][success] (true) + */ + bool ResizeBuffer( const std::streamsize bytes ); + + + // The size of _buffer. Default 4096. Call ResizeBuffer() to change the size. + const std::streamsize &buffer_size; // = _buffer_size + + // Error. Functions will not succeed when this is true. Call Close() or Dissociate(). + const bool &error; // = _error + + // Contains an error message when 'error'. + // If 'error' you can read this string before calling Close() or Dissociate(). + const std::string &error_msg; // = _error_msg + + // Set this to change the delimiter. + // The delimiter should be a single character but prepended/appended whitespace is ok. + // The delimiter is persistent and will survive resets. It doesn't need to be set on each open. + std::string delimiter; // = , + + // Set this to change the terminator. + // The terminator should be a single character or \r\n but prepended/appended whitespace is ok. + // \n could be automatically translated and written as \r\n in text mode, depending on your OS. + // It's not recommended to set the terminator as \r\n explicitly. + // The terminator is persistent and will survive resets. It doesn't need to be set on each open. + std::string terminator; // = \n + +private: + CSVwrite( const CSVwrite & ); + CSVwrite & operator=( const CSVwrite & ); + + // A file stream if one was opened by this class. + std::ofstream _file; + + // The stream the records are written to. + // This points to the user specified ostream or _file. + std::ostream *_output_ptr; + + // The flags passed to Open()/Associate(). + Flags _flags; + + // The buffer used to hold data written to the stream. + char *_buffer; + + // Whether or not the field to be written is the first field in the record. + bool _is_first_field; + + // For a description of any of these refer to their public const references. + std::streamsize _buffer_size; + bool _error; + std::string _error_msg; + + // Initialization to be called from the constructor only. + bool Init(); + + // Resets most variables. Does not reset the buffer size, delimiter or terminator. + bool Reset(); +}; + +inline CSVwrite::Flags operator | (CSVwrite::Flags a, CSVwrite::Flags b) +{ + return CSVwrite::Flags( ( (int)a ) | ( (int)b ) ); +} +inline CSVwrite::Flags & operator |= (CSVwrite::Flags &a, CSVwrite::Flags b) +{ + return (CSVwrite::Flags &)( ( (int &)a ) |= ( (int)b ) ); +} + + +} // namespace util +} // namespace jay +#endif // JAY_UTIL_CSV_HPP_ diff --git a/src/csv/CSVread.cpp b/src/csv/CSVread.cpp new file mode 100644 index 0000000000..ee1674db7a --- /dev/null +++ b/src/csv/CSVread.cpp @@ -0,0 +1,764 @@ +/* +Copyright (C) 2014 Jay Satiro +All rights reserved. + +This file is part of CSV/jay::util. + +https://github.com/jay/CSV + +jay::util is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +jay::util is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with jay::util. If not, see . +*/ + +/** A class to read comma separated values (CSV) from a stream. + +Documentation is in CSV.hpp. +*/ + +#include "CSV.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +#include "csv.h" + +#include "strerror.hpp" + + +using namespace std; + + +namespace jay { +namespace util { + + +struct cb_stuff +{ + cb_stuff( + list> &_cache, + CSVread::Flags &_flags, + bool &_error_pending, + std::string &_error_msg, + bool &_end_record_not_terminated, + uintmax_t &pending, + uintmax_t &requested + ) : + _cache( _cache ), _flags( _flags ), _error_pending( _error_pending ), _error_msg( _error_msg ), + _end_record_not_terminated( _end_record_not_terminated ), + pending( pending ), requested( requested ) + { + } + + // A reference to the CSVread::_cache list. + list> &_cache; + + // A reference to the CSVread::_flags. + const CSVread::Flags &_flags; + + // A reference to CSVread::_error_pending. + bool &_error_pending; + + // A reference to CSVread::_error_msg. + std::string &_error_msg; + + // A reference to CSVread::_end_record_not_terminated. + bool &_end_record_not_terminated; + + // A reference to the record number of the pending record. + uintmax_t &pending; + + // A reference to the record number of the requested record. + const uintmax_t &requested; + +private: + cb_stuff( const cb_stuff & ); + cb_stuff & operator=( const cb_stuff & ); +}; + + +// Called by libcsv when a new field has been parsed for the pending record. +// The pending record is the back element in the cache list. If the record number of the pending +// record is less than the record number of the requested record then it's ignored. +static void Callback_Field( void *data, size_t data_size, void *userptr ) +{ + cb_stuff *s = (cb_stuff *)userptr; + + if( s->_error_pending ) + { + return; + } + + if( s->pending >= s->requested ) + { + // Push the field to the back of the pending record. + s->_cache.back().push_back( string( (const char *)data, data_size ) ); + + if( ( s->_flags & CSVread::error_on_null_in_field ) + && ( s->_cache.back().back().find_first_of( '\0' ) != string::npos ) + ) + { + s->_error_pending = true; + ostringstream ss; + ss << "Record #" << s->pending << " Field #" << s->_cache.back().size() + << " is invalid due to NULL byte in field."; + s->_error_msg = ss.str(); + return; + } + } +} + + +// Called by libcsv when the pending record is complete (no more fields to parse). +// The pending record is the back element in the cache list. If the record number of the pending +// record is less than the record number of the requested record then it's ignored. +static void Callback_Record( int terminator, void *userptr ) +{ + cb_stuff *s = (cb_stuff *)userptr; + + if( s->_error_pending + || ( ( terminator == CSV_CR ) && ( s->_flags & CSVread::process_empty_records ) ) + ) + { + return; + } + + if( terminator == -1 ) + { + s->_end_record_not_terminated = true; + } + + if( s->pending >= s->requested ) + { + // Push a vector to the back of the list to make a new pending record. + s->_cache.push_back( vector() ); + } + + ++s->pending; +} + + +CSVread::CSVread() : + buffer_size( _buffer_size), eof( _eof ), error( _error ), error_msg( _error_msg ), + has_utf8_bom( _has_utf8_bom), record_num( _record_num ), end_record_num( _end_record_num ), + end_record_not_terminated( _end_record_not_terminated ), fields( _fields ) +{ + if( !Init() ) + return; +} + + +CSVread::CSVread( string filename, Flags flags /* = none */ ) : + buffer_size( _buffer_size), eof( _eof ), error( _error ), error_msg( _error_msg ), + has_utf8_bom( _has_utf8_bom), record_num( _record_num ), end_record_num( _end_record_num ), + end_record_not_terminated( _end_record_not_terminated ), fields( _fields ) +{ + if( !Init() ) + return; + + Open( filename, flags ); +} + + +CSVread::CSVread( istream *stream, Flags flags /* = none */ ) : + buffer_size( _buffer_size), eof( _eof ), error( _error ), error_msg( _error_msg ), + has_utf8_bom( _has_utf8_bom), record_num( _record_num ), end_record_num( _end_record_num ), + end_record_not_terminated( _end_record_not_terminated ), fields( _fields ) +{ + if( !Init() ) + return; + + Associate( stream, flags ); +} + + +CSVread::~CSVread() +{ + if( parse_obj ) + { + csv_free( parse_obj ); + delete parse_obj; + } + free( _buffer ); +} + + +/* Resize a buffer. + +Both CSVread and CSVwrite use this to resize their respective buffers. + +[ret][failure] (false) : The buffer could not be resized and has retained its current size. + 'error' and 'error_msg' are set. +[ret][success] (true) +*/ +bool CSVshared_ResizeBuffer( + const streamsize bytes, // IN + char *&_buffer, // INOUT + streamsize &_buffer_size, // INOUT + bool &_error, // OUT + string &_error_msg // OUT +) +{ + if( _buffer && ( _buffer_size == bytes ) ) + return true; + + if( bytes <= 0 ) + { + _error = true; + _error_msg = "buffer allocation failed. size of bytes is <= 0."; + return false; + } + + // CSV classes may cast 'buffer_size' to a size_t at any point so it can't be larger than that. + if( bytes > (std::numeric_limits::max)() ) + { + _error = true; + _error_msg = "buffer allocation failed. size of bytes is > std::numeric_limits::max"; + return false; + } + + char *temp = (char *)realloc( _buffer, (size_t)bytes ); + if( !temp ) + { + _error = true; + + ostringstream ss; + ss << "buffer allocation failed. size in bytes: " << bytes; + _error_msg = ss.str(); + + return false; + } + + _buffer = temp; + _buffer_size = bytes; + return true; +} + + +bool CSVread::ResizeBuffer( const streamsize bytes ) +{ + return CSVshared_ResizeBuffer( bytes, _buffer, _buffer_size, _error, _error_msg ); +} + + +bool CSVread::ResetParser() +{ + /* libcsv resets the parser in csv_fini() but not in csv_free(). If there is an error in the + parser object then csv_fini() may not have been called or if it was it may not have reset. + Also libcsv's manual advises against calling csv_init() on a parse object more than once: + "memory allocated for the original structure will be lost" + Probably if any allocated memory is freed first then it can be reinitialized without leak. + But instead to be safer I'm recreating parse_obj. + */ + if( parse_obj ) + { + csv_free( parse_obj ); + delete parse_obj; + parse_obj = NULL; + } + + parse_obj = new csv_parser; + if( csv_init( parse_obj, 0 /* Options must be set in Associate(), not here! */ ) ) + { + _error = true; + _error_msg = "libcsv: "; + _error_msg += csv_strerror( csv_error( parse_obj ) ); + + delete parse_obj; + parse_obj = NULL; + + return false; + } + + SetDelimiter( _delimiter ); + return true; +} + + +void CSVread::ResetCache() +{ + _cache = list>( 1, vector() ); +} + + +// REM this function is also called by Init() for initialization. +bool CSVread::Reset( bool partial_reset /* = false */ ) +{ + if( !_buffer ) + { + if( !ResizeBuffer( 4096 ) ) + return false; + } + + if( !ResetParser() ) + return false; + + ResetCache(); + + if( _input_ptr ) + { + _input_ptr->clear(); + _input_ptr->seekg( _has_utf8_bom ? 3 : 0 ); + _eof = _input_ptr->eof(); + } + else + { + _flags = CSVread::none; + _eof = false; + _has_utf8_bom = false; + } + + csv_set_opts( parse_obj, ( ( _flags & process_empty_records ) ? CSV_REPALL_NL : 0 ) + | ( ( _flags & strict_mode ) ? ( CSV_STRICT | CSV_STRICT_FINI ) : 0 ) + ); + + _error = false; + _error_pending = false; + _error_msg = ""; + + if( !partial_reset ) + { + _record_num = 0; + _end_record_num = 0; + _end_record_not_terminated = false; + _fields = vector(); + } + + return true; +} + + +bool CSVread::Close() +{ + if( _file.is_open() ) + { + _file.close(); + } + + _input_ptr = NULL; + + return Reset(); +} + + +// Initialization to be called from the constructor only, first thing. +bool CSVread::Init() +{ + // The vars here must be zeroed before any possible error aborts the initialization. + _buffer = NULL; + _buffer_size = 0; + parse_obj = NULL; + _input_ptr = NULL; + + _delimiter = (unsigned char)CSV_COMMA; + + return Reset(); +} + + + + +bool CSVread::Associate( istream *stream, const Flags flags /* = none */ ) +{ + if( _error ) + return false; + + if( !stream ) + { + _error = true; + _error_msg = "The stream parameter is NULL."; + return false; + } + + if( _input_ptr ) + { + _error = true; + _error_msg = "A stream is already associated. Call Close() to dissociate."; + return false; + } + + if( ( flags & text_mode ) && ( stream != &_file ) ) + { + // For the time being text mode is only valid if it's a file opened by this class. + _error = true; + _error_msg = "Text mode is only valid for files opened by this class."; + return false; + } + + _flags = flags; + _input_ptr = stream; + + if( !_input_ptr->good() ) + { + _error = true; + _error_msg = "istream: " + ios_strerror( _input_ptr->rdstate() ); + return false; + } + + csv_set_opts( parse_obj, ( ( _flags & process_empty_records ) ? CSV_REPALL_NL : 0 ) + | ( ( _flags & strict_mode ) ? ( CSV_STRICT | CSV_STRICT_FINI ) : 0 ) + ); + + uintmax_t pending = 1; + uintmax_t requested = 1; + + bool parsed_end_record = false; + + cb_stuff args( _cache, _flags, _error_pending, _error_msg, _end_record_not_terminated, pending, requested ); + + /* At least 3 bytes need to be read to detect the UTF-8 BOM. If the _buffer has a size of less + than 3 then use temporary buffer a[] instead. + */ + char a[ 3 ]; + char *p = ( _buffer_size >= 3 ) ? _buffer : a; + streamsize p_size = ( _buffer_size >= 3 ) ? _buffer_size : (streamsize)sizeof a; + + _input_ptr->read( p, p_size ); + streamsize len = _input_ptr->gcount(); + _eof = _input_ptr->eof(); + + if( len > 0 ) + { + if( !( _flags & skip_utf8_bom_check ) + && ( len >= 3 ) + && ( p[ 0 ] == '\xEF' ) && ( p[ 1 ] == '\xBB' ) && ( p[ 2 ] == '\xBF' ) + ) + { + _has_utf8_bom = true; + } + + if( !_has_utf8_bom || ( len > 3 ) ) + { + char *adjusted_p = &p[ _has_utf8_bom ? 3 : 0 ]; + size_t adjusted_len = (size_t)( _has_utf8_bom ? ( len - 3 ) : len ); + + // REM the callbacks can modify most of the 'args' + if( csv_parse( + parse_obj, + adjusted_p, + adjusted_len, + Callback_Field, + Callback_Record, + &args + ) != adjusted_len + ) + { + if( !_error_pending ) + { + _error_pending = true; + _error_msg = "libcsv: "; + _error_msg += csv_strerror( csv_error( parse_obj ) ); + } + } + } + } + + // REM this block of code is duplicated in ReadRecord() + if( !_error_pending ) + { + if( !_input_ptr->good() || ( len != p_size ) ) + { + // REM the callbacks can modify most of the 'args' + if( csv_fini( parse_obj, Callback_Field, Callback_Record, &args ) ) + { + _error_msg = "libcsv: "; + _error_msg += csv_strerror( csv_error( parse_obj ) ); + } + else + { + _error_msg = "istream: " + ios_strerror( _input_ptr->rdstate() ); + + if( _eof ) + { + parsed_end_record = true; + } + } + + _error_pending = true; + } + } + + // REM this block of code is duplicated in ReadRecord() + if( parsed_end_record ) + { + _end_record_num = pending - 1; + // _end_record_not_terminated is handled via csv_fini() @ Callback_Record() + } + else + { + _end_record_num = 0; + _end_record_not_terminated = false; + } + + return true; +} + + +bool CSVread::Open( string filename, const Flags flags /* = none */ ) +{ + if( _error ) + return false; + + if( _file.is_open() ) + { + _error = true; + _error_msg = "A file is already open. Call Close() to close the file."; + return false; + } + + if( _input_ptr ) + { + _error = true; + _error_msg = "A stream is already associated. Call Close() to dissociate."; + return false; + } + + ios::openmode mode = ( ( flags & text_mode ) ) ? ios::in : ios::binary; + + _file.open( filename, mode ); + if( !_file ) + { + _error = true; + _error_msg = "Failed opening " + filename; + return false; + } + + return Associate( &_file, flags ); +} + + + + +unsigned char CSVread::GetDelimiter() +{ + return _delimiter; +} + + +void CSVread::SetDelimiter( unsigned char delim ) +{ + _delimiter = delim; + + if( parse_obj ) + { + csv_set_delim( parse_obj, _delimiter ); + } +} + + + + +bool CSVread::ReadRecord( const uintmax_t requested_record_num /* = 0 */ ) +{ + if( _error ) + return false; + + if( !_input_ptr ) + { + _error = true; + _error_msg = "A stream is not associated with the object."; + return false; + } + + if( !_cache.size() ) + { + _error = true; + _error_msg = "The cache list does not contain any elements"; + return false; + } + + _eof = _input_ptr->eof(); + uintmax_t pending = _record_num + _cache.size(); + uintmax_t requested = requested_record_num; + + if( !requested ) + { + if( _record_num == SIZE_MAX ) + { + _error = true; + _error_msg = "The maximum number of lines have been read (SIZE_MAX)"; + return false; + } + requested = _record_num + 1; + } + + if( requested > _record_num ) + { + if( requested < pending ) // the requested record is already in the cache + { + for( uintmax_t i = requested - _record_num - 1; i; --i ) + { + _cache.pop_front(); + } + + _record_num = requested; + _fields.swap( _cache.front() ); + _cache.pop_front(); + return true; + } + else if( requested > pending ) // the requested record is not in the cache + { + // Discard all including the pending record. + ResetCache(); + } + else // the requested record is the pending record + { + // Discard all except the pending record. + vector temp; + temp.swap( _cache.back() ); + ResetCache(); + temp.swap( _cache.back() ); + } + } + else if( requested < _record_num ) + { + /* Records can span multiple lines and since the position of each record isn't exposed by + libcsv there's no way (short of modifying libcsv and keeping a separate cache) to seek + directly to the requested record's position. Instead do a partial reset here. A partial + reset does not change _fields, _record_num, _end_record_num and _end_record_not_terminated, + which are not to be changed unless this function is successful. + */ + + if( !Reset( true ) ) + { + // _error and _error_msg are already set by Reset() or its helpers if it failed. + return false; + } + + if( !_input_ptr->good() ) + { + _error = true; + _error_msg = "istream seek failed: " + ios_strerror( _input_ptr->rdstate() ); + return false; + } + + pending = 1; + } + else // requested == _record_num + { + return true; + } + + // At this point the cache does not have any complete records so error if an error is pending. + if( _error_pending ) + { + _error_pending = false; + _error = true; + return false; + } + + if( _cache.size() != 1 ) + { + _error = true; + + ostringstream ss; + ss << "The cache list has an unexpected size: " << _cache.size(); + _error_msg = ss.str(); + + return false; + } + + if( !_input_ptr->good() ) + { + _error = true; + _error_msg = "istream: " + ios_strerror( _input_ptr->rdstate() ); + return false; + } + + bool parsed_end_record = false; + + cb_stuff args( _cache, _flags, _error_pending, _error_msg, _end_record_not_terminated, pending, requested ); + + while( ( _cache.size() == 1 ) && !_error_pending ) + { + _input_ptr->read( _buffer, _buffer_size ); + streamsize len = _input_ptr->gcount(); + _eof = _input_ptr->eof(); + + if( len > 0 ) + { + // REM the callbacks can modify most of the 'args' + if( csv_parse( parse_obj, _buffer, (size_t)len, Callback_Field, Callback_Record, &args ) != len ) + { + if( !_error_pending ) + { + _error_pending = true; + _error_msg = "libcsv: "; + _error_msg += csv_strerror( csv_error( parse_obj ) ); + } + } + } + + // REM this block of code is duplicated in Associate() + if( !_error_pending ) + { + if( !_input_ptr->good() || ( len != _buffer_size ) ) + { + // REM the callback can modify most of the 'args' + if( csv_fini( parse_obj, Callback_Field, Callback_Record, &args ) ) + { + _error_msg = "libcsv: "; + _error_msg += csv_strerror( csv_error( parse_obj ) ); + } + else + { + _error_msg = "istream: " + ios_strerror( _input_ptr->rdstate() ); + + if( _eof ) + { + parsed_end_record = true; + } + } + + _error_pending = true; + } + } + } + + // REM this block of code is duplicated in Associate() + if( parsed_end_record ) + { + _end_record_num = pending - 1; + // _end_record_not_terminated is handled via csv_fini() @ Callback_Record() + } + else + { + _end_record_num = 0; + _end_record_not_terminated = false; + } + + if( _cache.size() == 1 ) + { + _error = true; + if( !_error_pending ) + { + _error_msg = "istream: " + ios_strerror( _input_ptr->rdstate() ); + } + return false; + } + + _record_num = requested; + _fields.swap( _cache.front() ); + _cache.pop_front(); + + return true; +} + + +} // namespace util +} // namespace jay diff --git a/src/csv/CSVwrite.cpp b/src/csv/CSVwrite.cpp new file mode 100644 index 0000000000..5f5ded2d0a --- /dev/null +++ b/src/csv/CSVwrite.cpp @@ -0,0 +1,440 @@ +/* +Copyright (C) 2014 Jay Satiro +All rights reserved. + +This file is part of CSV/jay::util. + +https://github.com/jay/CSV + +jay::util is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +jay::util is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with jay::util. If not, see . +*/ + +/** A class to write comma separated values (CSV) to a stream. + +Documentation is in CSV.hpp. +*/ + +#include "CSV.hpp" + +#include +#include +#include +#include +#include + +#include "strerror.hpp" + + +using namespace std; + + +namespace jay { +namespace util { + + +CSVwrite::CSVwrite() : + buffer_size( _buffer_size), error( _error ), error_msg( _error_msg ) +{ + if( !Init() ) + return; +} + + +CSVwrite::CSVwrite( string filename, Flags flags /* = none */ ) : + buffer_size( _buffer_size), error( _error ), error_msg( _error_msg ) +{ + if( !Init() ) + return; + + Open( filename, flags ); +} + + +CSVwrite::CSVwrite( ostream *stream, Flags flags /* = none */ ) : + buffer_size( _buffer_size), error( _error ), error_msg( _error_msg ) +{ + if( !Init() ) + return; + + Associate( stream, flags ); +} + + +CSVwrite::~CSVwrite() +{ + free( _buffer ); +} + + +bool CSVshared_ResizeBuffer( + const streamsize bytes, // IN + char *&buffer, // INOUT + streamsize &buffer_size, // INOUT + bool &error, // OUT + string &error_msg // OUT +); // This function defined and documented above CSVread::ResizeBuffer(). + + +bool CSVwrite::ResizeBuffer( const streamsize bytes ) +{ + return CSVshared_ResizeBuffer( bytes, _buffer, _buffer_size, _error, _error_msg ); +} + + +// REM This function is also called by Init() for initialization +bool CSVwrite::Reset() +{ + if( _output_ptr ) + { + _error = true; + _error_msg = "Not implemented"; + return false; + } + + if( !_buffer ) + { + if( !ResizeBuffer( 4096 ) ) + return false; + } + + _flags = CSVwrite::none; + _error = false; + _error_msg = ""; + _is_first_field = true; + + return true; +} + + +bool CSVwrite::Close() +{ + if( _file.is_open() ) + { + _file.close(); + } + + _output_ptr = NULL; + + return Reset(); +} + + +// Initialization to be called from the constructor only, first thing. +bool CSVwrite::Init() +{ + // The vars here must be zeroed before any possible error aborts the initialization. + _buffer = NULL; + _buffer_size = 0; + _output_ptr = NULL; + + delimiter = ","; + terminator = "\n"; + + return Reset(); +} + + + + +bool CSVwrite::Associate( ostream *stream, const Flags flags /* = none */ ) +{ + if( _error ) + return false; + + if( !stream ) + { + _error = true; + _error_msg = "The stream parameter is NULL."; + return false; + } + + if( _output_ptr ) + { + _error = true; + _error_msg = "A stream is already associated. Call Close() to dissociate."; + return false; + } + + if( ( flags & text_mode ) && ( stream != &_file ) ) + { + // For the time being text mode is only valid if it's a file opened by this class. + _error = true; + _error_msg = "Text mode is only valid for files opened by this class."; + return false; + } + + _flags = flags; + _output_ptr = stream; + + if( !_output_ptr->good() ) + { + _error = true; + _error_msg = "ostream: " + ios_strerror( _output_ptr->rdstate() ); + return false; + } + + return true; +} + + +bool CSVwrite::Open( string filename, const Flags flags /* = none */ ) +{ + if( _error ) + return false; + + if( _file.is_open() ) + { + _error = true; + _error_msg = "A file is already open. Call Close() to close the file."; + return false; + } + + if( _output_ptr ) + { + _error = true; + _error_msg = "A stream is already associated. Call Close() to dissociate."; + return false; + } + + ios::openmode mode = ( ( flags & text_mode ) ) ? ios::out : ios::binary; + mode |= ( ( flags & truncate ) ) ? ios::trunc : ios::app; + + _file.open( filename, mode ); + if( !_file ) + { + _error = true; + _error_msg = "Failed opening " + filename; + return false; + } + + return Associate( &_file, flags ); +} + + + + +bool CSVwrite::WriteUTF8BOM() +{ + if( _error ) + return false; + + if( !_output_ptr ) + { + _error = true; + _error_msg = "A stream is not associated with the object."; + return false; + } + + _output_ptr->write( "\xEF\xBB\xBF", 3 ); + if( !_output_ptr->good() ) + { + _error = true; + _error_msg = "ostream: " + ios_strerror( _output_ptr->rdstate() ); + return false; + } + + return true; +} + + +bool CSVwrite::WriteTerminator() +{ + if( _error ) + return false; + + if( !_output_ptr ) + { + _error = true; + _error_msg = "A stream is not associated with the object."; + return false; + } + + _output_ptr->write( terminator.c_str(), terminator.length() ); + if( !_output_ptr->good() ) + { + _error = true; + _error_msg = "ostream: " + ios_strerror( _output_ptr->rdstate() ); + return false; + } + + _is_first_field = true; + return true; +} + + +bool CSVwrite::WriteField( const string &field, bool terminate /* = false */ ) +{ + if( _error ) + return false; + + if( !_output_ptr ) + { + _error = true; + _error_msg = "A stream is not associated with the object."; + return false; + } + + char *p = _buffer; + const char *const buffer_end = _buffer + (size_t)_buffer_size; + + list prepend, append; + + if( !_is_first_field ) + { + prepend.assign( delimiter.begin(), delimiter.end() ); + } + + if( terminate ) + { + append.assign( terminator.begin(), terminator.end() ); + } + + // All fields are qualified with double quotes since that is what libcsv write functions do + prepend.push_back( '"' ); + append.push_front( '"' ); + + // This is set when there is a quote that needs to be escaped but there's no room in the buffer + bool quote = false; + + // This is set when the buffer needs to be written to the stream even if the buffer is not full + bool flush = false; + + for( string::const_iterator it = field.begin();; ) + { + if( flush || ( p == buffer_end ) ) + { + _output_ptr->write( _buffer, ( p - _buffer ) ); + if( !_output_ptr->good() ) + { + _error = true; + _error_msg = "ostream: " + ios_strerror( _output_ptr->rdstate() ); + return false; + } + + if( flush && ( it == field.end() ) ) + { + break; + } + + p = _buffer; + } + + if( it == field.begin() ) + { + while( prepend.size() && ( p != buffer_end ) ) + { + *p++ = *prepend.begin(); + prepend.pop_front(); + } + + if( p == buffer_end ) + { + continue; + } + } + + if( quote ) + { + *p++ = '"'; + quote = false; + + if( p == buffer_end ) + { + continue; + } + } + + if( it == field.end() ) + { + while( append.size() && ( p != buffer_end ) ) + { + *p++ = *append.begin(); + append.pop_front(); + } + + if( !append.size() || ( p != buffer_end ) ) + { + flush = true; + } + + continue; + } + + while( ( p != buffer_end ) && ( it != field.end() ) ) + { + *p = *it++; + if( *p++ == '"' ) + { + if( p == buffer_end ) + { + quote = true; + break; + } + + *p++ = '"'; + } + } + } + + _is_first_field = terminate; + return true; +} + +bool CSVwrite::WriteRecord( const vector &fields, bool terminate /* = true */ ) +{ + if( _error ) + return false; + + if( !_output_ptr ) + { + _error = true; + _error_msg = "A stream is not associated with the object."; + return false; + } + + if( !_is_first_field ) + { + if( !WriteTerminator() ) + { + return false; + } + } + + if( !fields.size() && !(( _flags & CSVwrite::process_empty_records )) ) + { + return true; + } + + for( vector::const_iterator it = fields.begin(); it != fields.end(); ++it ) + { + if( !WriteField( *it ) ) + { + return false; + } + } + + if( terminate ) + { + if( !WriteTerminator() ) + { + return false; + } + } + + return true; +} + + +} // namespace util +} // namespace jay diff --git a/src/csv/License_GPLv3.txt b/src/csv/License_GPLv3.txt new file mode 100644 index 0000000000..94a9ed024d --- /dev/null +++ b/src/csv/License_GPLv3.txt @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/src/csv/csv.h b/src/csv/csv.h new file mode 100644 index 0000000000..3fbd3dd272 --- /dev/null +++ b/src/csv/csv.h @@ -0,0 +1,88 @@ +#ifndef LIBCSV_H__ +#define LIBCSV_H__ +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define CSV_MAJOR 3 +#define CSV_MINOR 0 +#define CSV_RELEASE 3 + +/* Error Codes */ +#define CSV_SUCCESS 0 +#define CSV_EPARSE 1 /* Parse error in strict mode */ +#define CSV_ENOMEM 2 /* Out of memory while increasing buffer size */ +#define CSV_ETOOBIG 3 /* Buffer larger than SIZE_MAX needed */ +#define CSV_EINVALID 4 /* Invalid code,should never be received from csv_error*/ + + +/* parser options */ +#define CSV_STRICT 1 /* enable strict mode */ +#define CSV_REPALL_NL 2 /* report all unquoted carriage returns and linefeeds */ +#define CSV_STRICT_FINI 4 /* causes csv_fini to return CSV_EPARSE if last + field is quoted and doesn't containg ending + quote */ +#define CSV_APPEND_NULL 8 /* Ensure that all fields are null-terminated */ +#define CSV_EMPTY_IS_NULL 16 /* Pass null pointer to cb1 function when + empty, unquoted fields are encountered */ + + +/* Character values */ +#define CSV_TAB 0x09 +#define CSV_SPACE 0x20 +#define CSV_CR 0x0d +#define CSV_LF 0x0a +#define CSV_COMMA 0x2c +#define CSV_QUOTE 0x22 + +struct csv_parser { + int pstate; /* Parser state */ + int quoted; /* Is the current field a quoted field? */ + size_t spaces; /* Number of continious spaces after quote or in a non-quoted field */ + unsigned char * entry_buf; /* Entry buffer */ + size_t entry_pos; /* Current position in entry_buf (and current size of entry) */ + size_t entry_size; /* Size of entry buffer */ + int status; /* Operation status */ + unsigned char options; + unsigned char quote_char; + unsigned char delim_char; + int (*is_space)(unsigned char); + int (*is_term)(unsigned char); + size_t blk_size; + void *(*malloc_func)(size_t); + void *(*realloc_func)(void *, size_t); + void (*free_func)(void *); +}; + +/* Function Prototypes */ +int csv_init(struct csv_parser *p, unsigned char options); +int csv_fini(struct csv_parser *p, void (*cb1)(void *, size_t, void *), void (*cb2)(int, void *), void *data); +void csv_free(struct csv_parser *p); +int csv_error(struct csv_parser *p); +char * csv_strerror(int error); +size_t csv_parse(struct csv_parser *p, const void *s, size_t len, void (*cb1)(void *, size_t, void *), void (*cb2)(int, void *), void *data); +size_t csv_write(void *dest, size_t dest_size, const void *src, size_t src_size); +int csv_fwrite(FILE *fp, const void *src, size_t src_size); +size_t csv_write2(void *dest, size_t dest_size, const void *src, size_t src_size, unsigned char quote); +int csv_fwrite2(FILE *fp, const void *src, size_t src_size, unsigned char quote); +int csv_get_opts(struct csv_parser *p); +int csv_set_opts(struct csv_parser *p, unsigned char options); +void csv_set_delim(struct csv_parser *p, unsigned char c); +void csv_set_quote(struct csv_parser *p, unsigned char c); +unsigned char csv_get_delim(struct csv_parser *p); +unsigned char csv_get_quote(struct csv_parser *p); +void csv_set_space_func(struct csv_parser *p, int (*f)(unsigned char)); +void csv_set_term_func(struct csv_parser *p, int (*f)(unsigned char)); +void csv_set_realloc_func(struct csv_parser *p, void *(*)(void *, size_t)); +void csv_set_free_func(struct csv_parser *p, void (*)(void *)); +void csv_set_blk_size(struct csv_parser *p, size_t); +size_t csv_get_buffer_size(struct csv_parser *p); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/csv/libcsv.c b/src/csv/libcsv.c new file mode 100644 index 0000000000..05b5c697f3 --- /dev/null +++ b/src/csv/libcsv.c @@ -0,0 +1,581 @@ +/* +libcsv - parse and write csv data +Copyright (C) 2008 Robert Gamble + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#if ___STDC_VERSION__ >= 199901L +# include +#else +# define SIZE_MAX ((size_t)-1) /* C89 doesn't have stdint.h or SIZE_MAX */ +#endif + +#include "csv.h" + +#define VERSION "3.0.3" + +#define ROW_NOT_BEGUN 0 +#define FIELD_NOT_BEGUN 1 +#define FIELD_BEGUN 2 +#define FIELD_MIGHT_HAVE_ENDED 3 + +/* + Explanation of states + ROW_NOT_BEGUN There have not been any fields encountered for this row + FIELD_NOT_BEGUN There have been fields but we are currently not in one + FIELD_BEGUN We are in a field + FIELD_MIGHT_HAVE_ENDED + We encountered a double quote inside a quoted field, the + field is either ended or the quote is literal +*/ + +#define MEM_BLK_SIZE 128 + +#define SUBMIT_FIELD(p) \ + do { \ + if (!quoted) \ + entry_pos -= spaces; \ + if (p->options & CSV_APPEND_NULL) \ + ((p)->entry_buf[entry_pos]) = '\0'; \ + if (cb1 && (p->options & CSV_EMPTY_IS_NULL) && !quoted && entry_pos == 0) \ + cb1(NULL, entry_pos, data); \ + else if (cb1) \ + cb1(p->entry_buf, entry_pos, data); \ + pstate = FIELD_NOT_BEGUN; \ + entry_pos = quoted = spaces = 0; \ + } while (0) + +#define SUBMIT_ROW(p, c) \ + do { \ + if (cb2) \ + cb2(c, data); \ + pstate = ROW_NOT_BEGUN; \ + entry_pos = quoted = spaces = 0; \ + } while (0) + +#define SUBMIT_CHAR(p, c) ((p)->entry_buf[entry_pos++] = (c)) + +static char *csv_errors[] = {"success", + "error parsing data while strict checking enabled", + "memory exhausted while increasing buffer size", + "data size too large", + "invalid status code"}; + +int +csv_error(struct csv_parser *p) +{ + /* Return the current status of the parser */ + return p->status; +} + +char * +csv_strerror(int status) +{ + /* Return a textual description of status */ + if (status >= CSV_EINVALID || status < 0) + return csv_errors[CSV_EINVALID]; + else + return csv_errors[status]; +} + +int +csv_get_opts(struct csv_parser *p) +{ + /* Return the currently set options of parser */ + if (p == NULL) + return -1; + + return p->options; +} + +int +csv_set_opts(struct csv_parser *p, unsigned char options) +{ + /* Set the options */ + if (p == NULL) + return -1; + + p->options = options; + return 0; +} + +int +csv_init(struct csv_parser *p, unsigned char options) +{ + /* Initialize a csv_parser object returns 0 on success, -1 on error */ + if (p == NULL) + return -1; + + p->entry_buf = NULL; + p->pstate = ROW_NOT_BEGUN; + p->quoted = 0; + p->spaces = 0; + p->entry_pos = 0; + p->entry_size = 0; + p->status = 0; + p->options = options; + p->quote_char = CSV_QUOTE; + p->delim_char = CSV_COMMA; + p->is_space = NULL; + p->is_term = NULL; + p->blk_size = MEM_BLK_SIZE; + p->malloc_func = NULL; + p->realloc_func = realloc; + p->free_func = free; + + return 0; +} + +void +csv_free(struct csv_parser *p) +{ + /* Free the entry_buffer of csv_parser object */ + if (p == NULL) + return; + + if (p->entry_buf) + p->free_func(p->entry_buf); + + p->entry_buf = NULL; + p->entry_size = 0; + + return; +} + +int +csv_fini(struct csv_parser *p, void (*cb1)(void *, size_t, void *), void (*cb2)(int c, void *), void *data) +{ + /* Finalize parsing. Needed, for example, when file does not end in a newline */ + int quoted = p->quoted; + int pstate = p->pstate; + size_t spaces = p->spaces; + size_t entry_pos = p->entry_pos; + + if (p == NULL) + return -1; + + + if (p->pstate == FIELD_BEGUN && p->quoted && p->options & CSV_STRICT && p->options & CSV_STRICT_FINI) { + /* Current field is quoted, no end-quote was seen, and CSV_STRICT_FINI is set */ + p->status = CSV_EPARSE; + return -1; + } + + switch (p->pstate) { + case FIELD_MIGHT_HAVE_ENDED: + p->entry_pos -= p->spaces + 1; /* get rid of spaces and original quote */ + /* Fall-through */ + case FIELD_NOT_BEGUN: + case FIELD_BEGUN: + quoted = p->quoted, pstate = p->pstate; + spaces = p->spaces, entry_pos = p->entry_pos; + SUBMIT_FIELD(p); + SUBMIT_ROW(p, -1); + case ROW_NOT_BEGUN: /* Already ended properly */ + ; + } + + /* Reset parser */ + p->spaces = p->quoted = p->entry_pos = p->status = 0; + p->pstate = ROW_NOT_BEGUN; + + return 0; +} + +void +csv_set_delim(struct csv_parser *p, unsigned char c) +{ + /* Set the delimiter */ + if (p) p->delim_char = c; +} + +void +csv_set_quote(struct csv_parser *p, unsigned char c) +{ + /* Set the quote character */ + if (p) p->quote_char = c; +} + +unsigned char +csv_get_delim(struct csv_parser *p) +{ + /* Get the delimiter */ + return p->delim_char; +} + +unsigned char +csv_get_quote(struct csv_parser *p) +{ + /* Get the quote character */ + return p->quote_char; +} + +void +csv_set_space_func(struct csv_parser *p, int (*f)(unsigned char)) +{ + /* Set the space function */ + if (p) p->is_space = f; +} + +void +csv_set_term_func(struct csv_parser *p, int (*f)(unsigned char)) +{ + /* Set the term function */ + if (p) p->is_term = f; +} + +void +csv_set_realloc_func(struct csv_parser *p, void *(*f)(void *, size_t)) +{ + /* Set the realloc function used to increase buffer size */ + if (p && f) p->realloc_func = f; +} + +void +csv_set_free_func(struct csv_parser *p, void (*f)(void *)) +{ + /* Set the free function used to free the buffer */ + if (p && f) p->free_func = f; +} + +void +csv_set_blk_size(struct csv_parser *p, size_t size) +{ + /* Set the block size used to increment buffer size */ + if (p) p->blk_size = size; +} + +size_t +csv_get_buffer_size(struct csv_parser *p) +{ + /* Get the size of the entry buffer */ + if (p) + return p->entry_size; + return 0; +} + +static int +csv_increase_buffer(struct csv_parser *p) +{ + /* Increase the size of the entry buffer. Attempt to increase size by + * p->blk_size, if this is larger than SIZE_MAX try to increase current + * buffer size to SIZE_MAX. If allocation fails, try to allocate halve + * the size and try again until successful or increment size is zero. + */ + + size_t to_add = p->blk_size; + void *vp; + + if ( p->entry_size >= SIZE_MAX - to_add ) + to_add = SIZE_MAX - p->entry_size; + + if (!to_add) { + p->status = CSV_ETOOBIG; + return -1; + } + + while ((vp = p->realloc_func(p->entry_buf, p->entry_size + to_add)) == NULL) { + to_add /= 2; + if (!to_add) { + p->status = CSV_ENOMEM; + return -1; + } + } + + /* Update entry buffer pointer and entry_size if successful */ + p->entry_buf = vp; + p->entry_size += to_add; + return 0; +} + +size_t +csv_parse(struct csv_parser *p, const void *s, size_t len, void (*cb1)(void *, size_t, void *), void (*cb2)(int c, void *), void *data) +{ + unsigned const char *us = s; /* Access input data as array of unsigned char */ + unsigned char c; /* The character we are currently processing */ + size_t pos = 0; /* The number of characters we have processed in this call */ + + /* Store key fields into local variables for performance */ + unsigned char delim = p->delim_char; + unsigned char quote = p->quote_char; + int (*is_space)(unsigned char) = p->is_space; + int (*is_term)(unsigned char) = p->is_term; + int quoted = p->quoted; + int pstate = p->pstate; + size_t spaces = p->spaces; + size_t entry_pos = p->entry_pos; + + + if (!p->entry_buf && pos < len) { + /* Buffer hasn't been allocated yet and len > 0 */ + if (csv_increase_buffer(p) != 0) { + p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; + return pos; + } + } + + while (pos < len) { + /* Check memory usage, increase buffer if neccessary */ + if (entry_pos == ((p->options & CSV_APPEND_NULL) ? p->entry_size - 1 : p->entry_size) ) { + if (csv_increase_buffer(p) != 0) { + p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; + return pos; + } + } + + c = us[pos++]; + + switch (pstate) { + case ROW_NOT_BEGUN: + case FIELD_NOT_BEGUN: + if ((is_space ? is_space(c) : c == CSV_SPACE || c == CSV_TAB) && c!=delim) { /* Space or Tab */ + continue; + } else if (is_term ? is_term(c) : c == CSV_CR || c == CSV_LF) { /* Carriage Return or Line Feed */ + if (pstate == FIELD_NOT_BEGUN) { + SUBMIT_FIELD(p); + SUBMIT_ROW(p, (unsigned char)c); + } else { /* ROW_NOT_BEGUN */ + /* Don't submit empty rows by default */ + if (p->options & CSV_REPALL_NL) { + SUBMIT_ROW(p, (unsigned char)c); + } + } + continue; + } else if (c == delim) { /* Comma */ + SUBMIT_FIELD(p); + break; + } else if (c == quote) { /* Quote */ + pstate = FIELD_BEGUN; + quoted = 1; + } else { /* Anything else */ + pstate = FIELD_BEGUN; + quoted = 0; + SUBMIT_CHAR(p, c); + } + break; + case FIELD_BEGUN: + if (c == quote) { /* Quote */ + if (quoted) { + SUBMIT_CHAR(p, c); + pstate = FIELD_MIGHT_HAVE_ENDED; + } else { + /* STRICT ERROR - double quote inside non-quoted field */ + if (p->options & CSV_STRICT) { + p->status = CSV_EPARSE; + p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; + return pos-1; + } + SUBMIT_CHAR(p, c); + spaces = 0; + } + } else if (c == delim) { /* Comma */ + if (quoted) { + SUBMIT_CHAR(p, c); + } else { + SUBMIT_FIELD(p); + } + } else if (is_term ? is_term(c) : c == CSV_CR || c == CSV_LF) { /* Carriage Return or Line Feed */ + if (!quoted) { + SUBMIT_FIELD(p); + SUBMIT_ROW(p, (unsigned char)c); + } else { + SUBMIT_CHAR(p, c); + } + } else if (!quoted && (is_space? is_space(c) : c == CSV_SPACE || c == CSV_TAB)) { /* Tab or space for non-quoted field */ + SUBMIT_CHAR(p, c); + spaces++; + } else { /* Anything else */ + SUBMIT_CHAR(p, c); + spaces = 0; + } + break; + case FIELD_MIGHT_HAVE_ENDED: + /* This only happens when a quote character is encountered in a quoted field */ + if (c == delim) { /* Comma */ + entry_pos -= spaces + 1; /* get rid of spaces and original quote */ + SUBMIT_FIELD(p); + } else if (is_term ? is_term(c) : c == CSV_CR || c == CSV_LF) { /* Carriage Return or Line Feed */ + entry_pos -= spaces + 1; /* get rid of spaces and original quote */ + SUBMIT_FIELD(p); + SUBMIT_ROW(p, (unsigned char)c); + } else if (is_space ? is_space(c) : c == CSV_SPACE || c == CSV_TAB) { /* Space or Tab */ + SUBMIT_CHAR(p, c); + spaces++; + } else if (c == quote) { /* Quote */ + if (spaces) { + /* STRICT ERROR - unescaped double quote */ + if (p->options & CSV_STRICT) { + p->status = CSV_EPARSE; + p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; + return pos-1; + } + spaces = 0; + SUBMIT_CHAR(p, c); + } else { + /* Two quotes in a row */ + pstate = FIELD_BEGUN; + } + } else { /* Anything else */ + /* STRICT ERROR - unescaped double quote */ + if (p->options & CSV_STRICT) { + p->status = CSV_EPARSE; + p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; + return pos-1; + } + pstate = FIELD_BEGUN; + spaces = 0; + SUBMIT_CHAR(p, c); + } + break; + default: + break; + } + } + p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; + return pos; +} + +size_t +csv_write (void *dest, size_t dest_size, const void *src, size_t src_size) +{ + unsigned char *cdest = dest; + const unsigned char *csrc = src; + size_t chars = 0; + + if (src == NULL) + return 0; + + if (cdest == NULL) + dest_size = 0; + + if (dest_size > 0) + *cdest++ = '"'; + chars++; + + while (src_size) { + if (*csrc == '"') { + if (dest_size > chars) + *cdest++ = '"'; + if (chars < SIZE_MAX) chars++; + } + if (dest_size > chars) + *cdest++ = *csrc; + if (chars < SIZE_MAX) chars++; + src_size--; + csrc++; + } + + if (dest_size > chars) + *cdest = '"'; + if (chars < SIZE_MAX) chars++; + + return chars; +} + +int +csv_fwrite (FILE *fp, const void *src, size_t src_size) +{ + const unsigned char *csrc = src; + + if (fp == NULL || src == NULL) + return 0; + + if (fputc('"', fp) == EOF) + return EOF; + + while (src_size) { + if (*csrc == '"') { + if (fputc('"', fp) == EOF) + return EOF; + } + if (fputc(*csrc, fp) == EOF) + return EOF; + src_size--; + csrc++; + } + + if (fputc('"', fp) == EOF) { + return EOF; + } + + return 0; +} + +size_t +csv_write2 (void *dest, size_t dest_size, const void *src, size_t src_size, unsigned char quote) +{ + unsigned char *cdest = dest; + const unsigned char *csrc = src; + size_t chars = 0; + + if (src == NULL) + return 0; + + if (dest == NULL) + dest_size = 0; + + if (dest_size > 0) + *cdest++ = quote; + chars++; + + while (src_size) { + if (*csrc == quote) { + if (dest_size > chars) + *cdest++ = quote; + if (chars < SIZE_MAX) chars++; + } + if (dest_size > chars) + *cdest++ = *csrc; + if (chars < SIZE_MAX) chars++; + src_size--; + csrc++; + } + + if (dest_size > chars) + *cdest = quote; + if (chars < SIZE_MAX) chars++; + + return chars; +} + +int +csv_fwrite2 (FILE *fp, const void *src, size_t src_size, unsigned char quote) +{ + const unsigned char *csrc = src; + + if (fp == NULL || src == NULL) + return 0; + + if (fputc(quote, fp) == EOF) + return EOF; + + while (src_size) { + if (*csrc == quote) { + if (fputc(quote, fp) == EOF) + return EOF; + } + if (fputc(*csrc, fp) == EOF) + return EOF; + src_size--; + csrc++; + } + + if (fputc(quote, fp) == EOF) { + return EOF; + } + + return 0; +} diff --git a/src/csv/strerror.cpp b/src/csv/strerror.cpp new file mode 100644 index 0000000000..0a68f8da42 --- /dev/null +++ b/src/csv/strerror.cpp @@ -0,0 +1,81 @@ +/* +Copyright (C) 2014 Jay Satiro +All rights reserved. + +This file is part of CSV/jay::util. + +https://github.com/jay/CSV + +jay::util is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +jay::util is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with jay::util. If not, see . +*/ + +/** Error to string functions +*/ + +#include "strerror.hpp" + +#include +#include +#include +#include +#include + + +using namespace std; + + +namespace jay { +namespace util { + + +string ios_strerror( ios::iostate state ) +{ + if( !state ) + return "No errors"; + + string s; + + if( ( state & ios::eofbit ) ) + { + if( s.length() ) + s += ", "; + + s += "EOF on input operation"; + } + + if( ( state & ios::failbit ) ) + { + if( s.length() ) + s += ", "; + + s += "Logical error on i/o operation"; + } + + if( ( state & ios::badbit ) ) + { + if( s.length() ) + s += ", "; + + s += "Read/write error on i/o operation"; + } + + if( !s.length() ) + s = "Unknown error"; + + return s; +} + + +} // namespace util +} // namespace jay diff --git a/src/csv/strerror.hpp b/src/csv/strerror.hpp new file mode 100644 index 0000000000..f4d2333ee8 --- /dev/null +++ b/src/csv/strerror.hpp @@ -0,0 +1,40 @@ +/* +Copyright (C) 2014 Jay Satiro +All rights reserved. + +This file is part of CSV/jay::util. + +https://github.com/jay/CSV + +jay::util is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +jay::util is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with jay::util. If not, see . +*/ + +#ifndef JAY_UTIL_STRERROR_HPP_ +#define JAY_UTIL_STRERROR_HPP_ + +#include +#include + + +namespace jay { +namespace util { + + +// Returns an iostate error message. +std::string ios_strerror( std::ios::iostate state ); + + +} // namespace util +} // namespace jay +#endif // JAY_UTIL_STRERROR_HPP_ diff --git a/src/net.cpp b/src/net.cpp index 48020caf73..979d56bf1c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -35,6 +35,7 @@ #include #endif +#include #include @@ -96,9 +97,9 @@ void CConnman::AddOneShot(const std::string& strDest) vOneShots.push_back(strDest); } -unsigned short GetListenPort() +uint16_t GetListenPort() { - return (unsigned short)(gArgs.GetArg("-port", Params().GetDefaultPort())); + return (uint16_t)(gArgs.GetArg("-port", Params().GetDefaultPort())); } // find 'best' local address for a particular peer diff --git a/src/net.h b/src/net.h index f9d1f1bff6..26188d355c 100644 --- a/src/net.h +++ b/src/net.h @@ -23,8 +23,8 @@ #include #include +#include #include -#include #include #include #include @@ -452,7 +452,7 @@ void Discover(); void StartMapPort(); void InterruptMapPort(); void StopMapPort(); -unsigned short GetListenPort(); +uint16_t GetListenPort(); bool BindListenPort(const CService &bindAddr, std::string& strError, bool fWhitelisted = false); struct CombinerAll diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 1b5b1ea182..360ef2f7ad 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -815,10 +815,11 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx due to expiration\n", nErased); } + FastRandomContext rng; while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: - uint256 randomhash = GetRandHash(); + uint256 randomhash = rng.rand256(); std::map::iterator it = mapOrphanTransactions.lower_bound(randomhash); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 4e25a147c9..9d5302db71 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include #include @@ -466,15 +467,15 @@ CService::CService() : port(0) { } -CService::CService(const CNetAddr& cip, unsigned short portIn) : CNetAddr(cip), port(portIn) +CService::CService(const CNetAddr& cip, uint16_t portIn) : CNetAddr(cip), port(portIn) { } -CService::CService(const struct in_addr& ipv4Addr, unsigned short portIn) : CNetAddr(ipv4Addr), port(portIn) +CService::CService(const struct in_addr& ipv4Addr, uint16_t portIn) : CNetAddr(ipv4Addr), port(portIn) { } -CService::CService(const struct in6_addr& ipv6Addr, unsigned short portIn) : CNetAddr(ipv6Addr), port(portIn) +CService::CService(const struct in6_addr& ipv6Addr, uint16_t portIn) : CNetAddr(ipv6Addr), port(portIn) { } @@ -502,7 +503,7 @@ bool CService::SetSockAddr(const struct sockaddr *paddr) } } -unsigned short CService::GetPort() const +uint16_t CService::GetPort() const { return port; } diff --git a/src/netaddress.h b/src/netaddress.h index 79f52e620f..02b089ebe5 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include @@ -146,10 +146,10 @@ class CService : public CNetAddr public: CService(); - CService(const CNetAddr& ip, unsigned short port); - CService(const struct in_addr& ipv4Addr, unsigned short port); + CService(const CNetAddr& ip, uint16_t port); + CService(const struct in_addr& ipv4Addr, uint16_t port); explicit CService(const struct sockaddr_in& addr); - unsigned short GetPort() const; + uint16_t GetPort() const; bool GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const; bool SetSockAddr(const struct sockaddr* paddr); friend bool operator==(const CService& a, const CService& b); @@ -160,7 +160,7 @@ class CService : public CNetAddr std::string ToStringPort() const; std::string ToStringIPPort() const; - CService(const struct in6_addr& ipv6Addr, unsigned short port); + CService(const struct in6_addr& ipv6Addr, uint16_t port); explicit CService(const struct sockaddr_in6& addr); ADD_SERIALIZE_METHODS; diff --git a/src/netbase.cpp b/src/netbase.cpp index 5249a3949d..423dbc1745 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -14,6 +14,7 @@ #include #include +#include #ifndef WIN32 #include @@ -600,11 +601,11 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, &random_auth, hSocket)) { return false; } } else { - if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, 0, hSocket)) { return false; } } diff --git a/src/optional.h b/src/optional.h new file mode 100644 index 0000000000..a382cd7b77 --- /dev/null +++ b/src/optional.h @@ -0,0 +1,26 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_OPTIONAL_H +#define BITCOIN_OPTIONAL_H + +#include + +#include + +//! Substitute for C++17 std::optional +template +using Optional = boost::optional; + +//! Substitute for C++17 std::make_optional +template +Optional MakeOptional(bool condition, T&& value) +{ + return boost::make_optional(condition, std::forward(value)); +} + +//! Substitute for C++17 std::nullopt +static auto& nullopt = boost::none; + +#endif // BITCOIN_OPTIONAL_H diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index cb6abb5ddd..e2e185f463 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -22,15 +22,7 @@ #include #ifdef WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 -#ifdef _WIN32_IE -#undef _WIN32_IE -#endif -#define _WIN32_IE 0x0501 -#define WIN32_LEAN_AND_MEAN 1 + #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/qt/macdockiconhandler.mm b/src/qt/macdockiconhandler.mm index 99f4573ebd..8c336f7023 100644 --- a/src/qt/macdockiconhandler.mm +++ b/src/qt/macdockiconhandler.mm @@ -27,18 +27,11 @@ bool dockClickHandler(id self,SEL _cmd,...) { } void setupDockClickHandler() { - Class cls = objc_getClass("NSApplication"); - id appInst = objc_msgSend((id)cls, sel_registerName("sharedApplication")); - if (appInst != nullptr) { - id delegate = objc_msgSend(appInst, sel_registerName("delegate")); - Class delClass = (Class)objc_msgSend(delegate, sel_registerName("class")); + Class delClass = (Class)[[[NSApplication sharedApplication] delegate] class]; + SEL shouldHandle = sel_registerName("applicationShouldHandleReopen:hasVisibleWindows:"); - if (class_getInstanceMethod(delClass, shouldHandle)) - class_replaceMethod(delClass, shouldHandle, (IMP)dockClickHandler, "B@:"); - else - class_addMethod(delClass, shouldHandle, (IMP)dockClickHandler,"B@:"); - } + class_replaceMethod(delClass, shouldHandle, (IMP)dockClickHandler, "B@:"); } diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 298cf75734..6d3f16797c 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_OPTIONSMODEL_H #include +#include #include @@ -18,7 +19,7 @@ class QNetworkProxy; QT_END_NAMESPACE extern const char *DEFAULT_GUI_PROXY_HOST; -static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; +static constexpr uint16_t DEFAULT_GUI_PROXY_PORT = 9050; /** Interface from Qt to configuration data structure for Veil client. To Qt, the options are presented as a list with the different options diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index d25cb59963..a181e8ed29 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include diff --git a/src/random.cpp b/src/random.cpp index ed18fb6c0e..d0403bc36c 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -397,6 +397,7 @@ uint256 FastRandomContext::rand256() std::vector FastRandomContext::randbytes(size_t len) { + if (requires_seed) RandomSeed(); std::vector ret(len); if (len > 0) { rng.Output(&ret[0], len); diff --git a/src/random.h b/src/random.h index f8de143f37..800dec3d18 100644 --- a/src/random.h +++ b/src/random.h @@ -130,6 +130,29 @@ class FastRandomContext { inline uint64_t operator()() { return rand64(); } }; +/** More efficient than using std::shuffle on a FastRandomContext. + * + * This is more efficient as std::shuffle will consume entropy in groups of + * 64 bits at the time and throw away most. + * + * This also works around a bug in libstdc++ std::shuffle that may cause + * type::operator=(type&&) to be invoked on itself, which the library's + * debug mode detects and panics on. This is a known issue, see + * https://stackoverflow.com/questions/22915325/avoiding-self-assignment-in-stdshuffle + */ +template +void Shuffle(I first, I last, R&& rng) +{ + while (first != last) { + size_t j = rng.randrange(last - first); + if (j) { + using std::swap; + swap(*first, *(first + j)); + } + ++first; + } +} + /* Number of random bytes returned by GetOSRand. * When changing this constant make sure to change all call sites, and make * sure that the underlying OS APIs for all platforms support the number. diff --git a/src/rest.cpp b/src/rest.cpp index 566e9b1b80..fc2ad05b88 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -173,7 +173,7 @@ static bool rest_headers(HTTPRequest* req, } case RetFormat::HEX: { - std::string strHex = HexStr(ssHeader.begin(), ssHeader.end()) + "\n"; + std::string strHex = HexStr(ssHeader) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -239,7 +239,7 @@ static bool rest_block(HTTPRequest* req, } case RetFormat::HEX: { - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()) + "\n"; + std::string strHex = HexStr(ssBlock) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -371,7 +371,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) } case RetFormat::HEX: { - std::string strHex = HexStr(ssTx.begin(), ssTx.end()) + "\n"; + std::string strHex = HexStr(ssTx) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -534,7 +534,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) case RetFormat::HEX: { CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; - std::string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; + std::string strHex = HexStr(ssGetUTXOResponse) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index b67c23215f..ad55b98f4c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -978,7 +978,7 @@ static UniValue getblockheader(const JSONRPCRequest& request) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); ssBlock << pblockindex->GetBlockHeader(); - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + std::string strHex = HexStr(ssBlock); return strHex; } @@ -1085,7 +1085,7 @@ static UniValue getblock(const JSONRPCRequest& request) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + std::string strHex = HexStr(ssBlock); return strHex; } @@ -2446,7 +2446,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) UniValue unspent(UniValue::VOBJ); unspent.pushKV("txid", outpoint.hash.GetHex()); unspent.pushKV("vout", (int32_t)outpoint.n); - unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey.begin(), txo.scriptPubKey.end())); + unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); unspent.pushKV("height", (int32_t)coin.nHeight); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 1646ee9da3..c71aeb30f3 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -896,7 +896,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) result.pushKV("height", (int64_t)(pindexPrev->nHeight+1)); if (!pblocktemplate->vchCoinbaseCommitment.empty() && fSupportsSegwit) { - result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment.begin(), pblocktemplate->vchCoinbaseCommitment.end())); + result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment)); } result.pushKV("accumulatorhashes", mapaccumulatorhashes); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index f3a279bab1..6cbf531eca 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -78,8 +78,8 @@ static UniValue validateaddress(const JSONRPCRequest& request) ret.pushKV("address", currentAddress); CScript scriptPubKey = GetScriptForDestination(dest); - ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); - + ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); + UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); } @@ -150,7 +150,7 @@ static UniValue createmultisig(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); result.pushKV("address", EncodeDestination(dest)); - result.pushKV("redeemScript", HexStr(inner.begin(), inner.end())); + result.pushKV("redeemScript", HexStr(inner)); return result; } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 313fd934f9..55496729d4 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -300,7 +300,7 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); CMerkleBlock mb(block, setTxids); ssMB << mb; - std::string strHex = HexStr(ssMB.begin(), ssMB.end()); + std::string strHex = HexStr(ssMB); return strHex; } @@ -1520,7 +1520,7 @@ UniValue decodepsbt(const JSONRPCRequest& request) if (!input.final_script_witness.IsNull()) { UniValue txinwitness(UniValue::VARR); for (const auto& item : input.final_script_witness.stack) { - txinwitness.push_back(HexStr(item.begin(), item.end())); + txinwitness.push_back(HexStr(item)); } in.pushKV("final_scriptwitness", txinwitness); } diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index c7a7602904..b5d6df770b 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -98,7 +98,7 @@ class DescribeAddressVisitor : public boost::static_visitor obj.pushKV("isscript", false); obj.pushKV("iswitness", true); obj.pushKV("witness_version", 0); - obj.pushKV("witness_program", HexStr(id.begin(), id.end())); + obj.pushKV("witness_program", HexStr(id)); return obj; } @@ -108,7 +108,7 @@ class DescribeAddressVisitor : public boost::static_visitor obj.pushKV("isscript", true); obj.pushKV("iswitness", true); obj.pushKV("witness_version", 0); - obj.pushKV("witness_program", HexStr(id.begin(), id.end())); + obj.pushKV("witness_program", HexStr(id)); return obj; } diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index e0d517533b..6bb1a7e08c 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -70,7 +70,7 @@ class ConstPubkeyProvider final : public PubkeyProvider } bool IsRange() const override { return false; } size_t GetSize() const override { return m_pubkey.size(); } - std::string ToString() const override { return HexStr(m_pubkey.begin(), m_pubkey.end()); } + std::string ToString() const override { return HexStr(m_pubkey); } bool ToPrivateString(const SigningProvider& arg, std::string& ret) const override { CKey key; diff --git a/src/serialize.h b/src/serialize.h index 19fad9a3ec..e5b009b115 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -10,12 +10,12 @@ #include #include +#include #include #include #include #include #include -#include #include #include #include @@ -284,7 +284,7 @@ inline void Unserialize(Stream& s, libzerocoin::SpendType & a) inline unsigned int GetSizeOfCompactSize(uint64_t nSize) { if (nSize < 253) return sizeof(unsigned char); - else if (nSize <= std::numeric_limits::max()) return sizeof(unsigned char) + sizeof(unsigned short); + else if (nSize <= std::numeric_limits::max()) return sizeof(unsigned char) + sizeof(uint16_t); else if (nSize <= std::numeric_limits::max()) return sizeof(unsigned char) + sizeof(unsigned int); else return sizeof(unsigned char) + sizeof(uint64_t); } @@ -298,7 +298,7 @@ void WriteCompactSize(Stream& os, uint64_t nSize) { ser_writedata8(os, nSize); } - else if (nSize <= std::numeric_limits::max()) + else if (nSize <= std::numeric_limits::max()) { ser_writedata8(os, 253); ser_writedata16(os, nSize); diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index c6a350bab7..e0137dcc32 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -10,11 +10,6 @@ #endif #ifdef WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 7cc864a9af..ed1b44f2ac 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -543,7 +543,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) CDataStream tmp(SER_DISK, CLIENT_VERSION); uint64_t x = 3000000000ULL; tmp << VARINT(x); - BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); + BOOST_CHECK_EQUAL(HexStr(tmp), "8a95c0bb00"); CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { Coin cc5; diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 02be899edf..2dbded5bdc 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -21,40 +21,23 @@ * using BOOST_CHECK_CLOSE to fail. * */ -FastRandomContext local_rand_ctx(true); - BOOST_AUTO_TEST_SUITE(cuckoocache_tests); - -/** insecure_GetRandHash fills in a uint256 from local_rand_ctx - */ -static void insecure_GetRandHash(uint256& t) -{ - uint32_t* ptr = (uint32_t*)t.begin(); - for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); -} - - - /* Test that no values not inserted into the cache are read out of it. * * There are no repeats in the first 200000 insecure_GetRandHash calls */ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); CuckooCache::cache cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); - uint256 v; for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - cc.insert(v); + cc.insert(InsecureRand256()); } for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - BOOST_CHECK(!cc.contains(v, false)); + BOOST_CHECK(!cc.contains(InsecureRand256(), false)); } }; @@ -64,7 +47,7 @@ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) template static double test_cache(size_t megabytes, double load) { - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); std::vector hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -74,7 +57,7 @@ static double test_cache(size_t megabytes, double load) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -135,7 +118,7 @@ template static void test_cache_erase(size_t megabytes) { double load = 1; - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); std::vector hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -145,7 +128,7 @@ static void test_cache_erase(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -198,7 +181,7 @@ template static void test_cache_erase_parallel(size_t megabytes) { double load = 1; - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); std::vector hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -208,7 +191,7 @@ static void test_cache_erase_parallel(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -300,7 +283,7 @@ static void test_cache_generations() // iterations with non-deterministic values, so it isn't "overfit" to the // specific entropy in FastRandomContext(true) and implementation of the // cache. - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); // block_activity models a chunk of network activity. n_insert elements are // added to the cache. The first and last n/4 are stored for removal later @@ -317,7 +300,7 @@ static void test_cache_generations() for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)inserts[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } for (uint32_t i = 0; i < n_insert / 4; ++i) reads.push_back(inserts[i]); diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index d1a535d689..b8d808edba 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) static void AddRandomOutboundPeer(std::vector &vNodes, PeerLogicValidation &peerLogic) { - CAddress addr(ip(GetRandInt(0xffffffff)), NODE_NONE); + CAddress addr(ip(insecure_rand_ctx.randbits(32)), NODE_NONE); vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); CNode &node = *vNodes.back(); node.SetSendVersion(PROTOCOL_VERSION); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 8be7a0b0c9..fb6738063d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -90,7 +90,7 @@ void Check(const std::string& prv, const std::string& pub, int flags, const std: BOOST_CHECK((t ? parse_priv : parse_pub)->Expand(i, key_provider, spks, script_provider)); BOOST_CHECK_EQUAL(spks.size(), ref.size()); for (size_t n = 0; n < spks.size(); ++n) { - BOOST_CHECK_EQUAL(ref[n], HexStr(spks[n].begin(), spks[n].end())); + BOOST_CHECK_EQUAL(ref[n], HexStr(spks[n])); BOOST_CHECK_EQUAL(IsSolvable(Merge(key_provider, script_provider), spks[n]), (flags & UNSOLVABLE) == 0); if (flags & SIGNABLE) { diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index df5c24b37d..09f413f68b 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -78,10 +79,10 @@ BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(cnode_listen_port) { // test default - unsigned short port = GetListenPort(); + uint16_t port = GetListenPort(); BOOST_CHECK(port == Params().GetDefaultPort()); // test set port - unsigned short altPort = 12345; + uint16_t altPort = 12345; gArgs.SoftSetArg("-port", std::to_string(altPort)); port = GetListenPort(); BOOST_CHECK(port == altPort); diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index d3278399a6..a4c3ce0f9a 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -209,8 +209,8 @@ class prevector_tester { prevector_tester() { SeedInsecureRand(); - rand_seed = insecure_rand_seed; - rand_cache = insecure_rand_ctx; + rand_seed = InsecureRand256(); + rand_cache = FastRandomContext(rand_seed); } }; diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 74e55b0322..b7aa138161 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -38,11 +38,18 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) BOOST_CHECK(ctx1.randbytes(50) == ctx2.randbytes(50)); // Check that a nondeterministic ones are not - FastRandomContext ctx3; - FastRandomContext ctx4; - BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal - BOOST_CHECK(ctx3.rand256() != ctx4.rand256()); - BOOST_CHECK(ctx3.randbytes(7) != ctx4.randbytes(7)); + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal + } + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.rand256() != ctx4.rand256()); + } + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.randbytes(7) != ctx4.randbytes(7)); + } } BOOST_AUTO_TEST_CASE(fastrandom_randbits) @@ -75,8 +82,42 @@ BOOST_AUTO_TEST_CASE(stdrandom_test) for (int j = 1; j <= 10; ++j) { BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); } + Shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } } } +/** Test that Shuffle reaches every permutation with equal probability. */ +BOOST_AUTO_TEST_CASE(shuffle_stat_test) +{ + FastRandomContext ctx(true); + uint32_t counts[5 * 5 * 5 * 5 * 5] = {0}; + for (int i = 0; i < 12000; ++i) { + int data[5] = {0, 1, 2, 3, 4}; + Shuffle(std::begin(data), std::end(data), ctx); + int pos = data[0] + data[1] * 5 + data[2] * 25 + data[3] * 125 + data[4] * 625; + ++counts[pos]; + } + unsigned int sum = 0; + double chi_score = 0.0; + for (int i = 0; i < 5 * 5 * 5 * 5 * 5; ++i) { + int i1 = i % 5, i2 = (i / 5) % 5, i3 = (i / 25) % 5, i4 = (i / 125) % 5, i5 = i / 625; + uint32_t count = counts[i]; + if (i1 == i2 || i1 == i3 || i1 == i4 || i1 == i5 || i2 == i3 || i2 == i4 || i2 == i5 || i3 == i4 || i3 == i5 || i4 == i5) { + BOOST_CHECK(count == 0); + } else { + chi_score += ((count - 100.0) * (count - 100.0)) / 100.0; + BOOST_CHECK(count > 50); + BOOST_CHECK(count < 150); + sum += count; + } + } + BOOST_CHECK(chi_score > 58.1411); // 99.9999% confidence interval + BOOST_CHECK(chi_score < 210.275); + BOOST_CHECK_EQUAL(sum, 12000); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 79549f2bc9..fbff012f4b 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(sighash_test) ss << txTo; std::cout << "\t[\"" ; - std::cout << HexStr(ss.begin(), ss.end()) << "\", \""; + std::cout << HexStr(ss) << "\", \""; std::cout << HexStr(scriptCode) << "\", "; std::cout << nIn << ", "; std::cout << nHashType << ", \""; diff --git a/src/test/test_veil.cpp b/src/test/test_veil.cpp index 317d89e206..7ea9371dd3 100644 --- a/src/test/test_veil.cpp +++ b/src/test/test_veil.cpp @@ -33,8 +33,7 @@ void CConnmanTest::ClearNodes() g_connman->vNodes.clear(); } -uint256 insecure_rand_seed = GetRandHash(); -FastRandomContext insecure_rand_ctx(insecure_rand_seed); +FastRandomContext insecure_rand_ctx; extern bool fPrintToConsole; extern void noui_connect(); diff --git a/src/test/test_veil.h b/src/test/test_veil.h index 6724dd4034..3e8d4c77ee 100644 --- a/src/test/test_veil.h +++ b/src/test/test_veil.h @@ -18,17 +18,18 @@ #include -extern uint256 insecure_rand_seed; +// Enable BOOST_CHECK_EQUAL for enum class types +template +std::ostream& operator<<(typename std::enable_if::value, std::ostream>::type& stream, const T& e) +{ + return stream << static_cast::type>(e); +} + extern FastRandomContext insecure_rand_ctx; -static inline void SeedInsecureRand(bool fDeterministic = false) +static inline void SeedInsecureRand(bool deterministic = false) { - if (fDeterministic) { - insecure_rand_seed = uint256(); - } else { - insecure_rand_seed = GetRandHash(); - } - insecure_rand_ctx = FastRandomContext(insecure_rand_seed); + insecure_rand_ctx = FastRandomContext(deterministic); } static inline uint32_t InsecureRand32() { return insecure_rand_ctx.rand32(); } diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 238b9f0880..931ed99a49 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -102,8 +102,8 @@ void BuildChain(const uint256& root, int height, const unsigned int invalid_rate { if (height <= 0 || blocks.size() >= max_size) return; - bool gen_invalid = GetRand(100) < invalid_rate; - bool gen_fork = GetRand(100) < branch_rate; + bool gen_invalid = InsecureRandRange(100) < invalid_rate; + bool gen_fork = InsecureRandRange(100) < branch_rate; const std::shared_ptr pblock = gen_invalid ? BadBlock(root) : GoodBlock(root); blocks.push_back(pblock); @@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) threads.create_thread([&blocks]() { bool ignored; for (int i = 0; i < 1000; i++) { - auto block = blocks[GetRand(blocks.size() - 1)]; + auto block = blocks[InsecureRandRange(blocks.size() - 1)]; ProcessNewBlock(Params(), block, true, &ignored); } diff --git a/src/util.cpp b/src/util.cpp index 7490bd11e9..4ccaa60bc7 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -44,17 +44,6 @@ #pragma warning(disable:4717) #endif -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 - -#ifdef _WIN32_IE -#undef _WIN32_IE -#endif -#define _WIN32_IE 0x0501 - -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/utiltime.cpp b/src/utiltime.cpp index c73051394c..f481c35a98 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -108,3 +108,61 @@ std::string FormatISO8601Time(int64_t nTime) { #endif return strprintf("%02i:%02i:%02iZ", ts.tm_hour, ts.tm_min, ts.tm_sec); } + +uint64_t ISO8601Date_Now() +{ + time_t rawtime; + time ( &rawtime ); + + return (uint64_t) rawtime; +} + +uint64_t ISO8601Date_FromString(const std::string& dateString) +{ + int year; + int month; + int day; + int hour; + int minute; + sscanf(dateString.c_str(), "%d-%d-%d:%d:%d", &year, &month, &day, &hour, &minute); + tm time; + time.tm_year = year - 1900; // Year since 1900 + time.tm_mon = month - 1; // 0-11 + time.tm_mday = day; // 1-31 + time.tm_hour = hour; // 0-23 + time.tm_min = minute; // 0-59 + time.tm_sec = 0; + time_t rawtime = mktime(&time); + return (uint64_t)rawtime; +} + +bool ISO8601Date_Validate(const std::string& dateString) +{ + int year; + int month; + int day; + int hour; + int minute; + sscanf(dateString.c_str(), "%d-%d-%d:%d:%d", &year, &month, &day, &hour, &minute); + if (year < 1900) + return false; + if (month < 1 || month > 12) + return false; + if (day < 1) + return false; + if ( (month == 1 || month == 3 || month == 5 || month == 7 || month == 9 || month == 11) && day > 31 ) + return false; + if ( (month == 4 || month == 6 || month == 8 || month == 10 || month == 12) && day > 30 ) + return false; + if ( (month == 2) && (year % 400 == 0 || (year % 4 == 0 && year % 100 != 0) && (day != 29) ) ) + return false; + else if ( month == 2 && day > 28) + return false; + if (hour > 24) + return false; + if (minute > 60) + return false; + + return true; +} + diff --git a/src/utiltime.h b/src/utiltime.h index 0286311401..9051237b69 100644 --- a/src/utiltime.h +++ b/src/utiltime.h @@ -34,5 +34,7 @@ void MilliSleep(int64_t n); std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); std::string FormatISO8601Time(int64_t nTime); - +uint64_t ISO8601Date_Now(); +uint64_t ISO8601Date_FromString(const std::string& dateString); +bool ISO8601Date_Validate(const std::string& dateString); #endif // BITCOIN_UTILTIME_H diff --git a/src/veil/ringct/anonwallet.cpp b/src/veil/ringct/anonwallet.cpp index db401b4a6d..f889f036b7 100644 --- a/src/veil/ringct/anonwallet.cpp +++ b/src/veil/ringct/anonwallet.cpp @@ -5883,7 +5883,7 @@ bool AnonWallet::SelectBlindedCoins(const std::vector &vAvailableCoins // add preset inputs to the total value selected nValueRet += nValueFromPresetInputs; - std::random_shuffle(setCoinsRet.begin(), setCoinsRet.end(), GetRandInt); + Shuffle(setCoinsRet.begin(), setCoinsRet.end(), FastRandomContext()); return res; } @@ -5972,7 +5972,7 @@ void AnonWallet::AvailableAnonCoins(std::vector &vCoins, bool fOnlySaf } } - random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); + Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); return; } @@ -6062,7 +6062,7 @@ bool AnonWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligi std::vector > > vValue; CAmount nTotalLower = 0; - random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); + Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); for (const auto &r : vCoins) { //if (!r.fSpendable) diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 7a27b878cd..bf7811baa0 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H +#include #include #include #include @@ -28,7 +29,7 @@ class CCoinControl //! Custom change destination, if not set an address is generated CTxDestination destChange; //! Override the default change type if set, ignored if destChange is set - boost::optional m_change_type; + Optional m_change_type; //! If false, allows unselected inputs, but requires all selected inputs be used bool fAllowOtherInputs; //! Includes watch only addresses which are solvable @@ -36,11 +37,11 @@ class CCoinControl //! Override automatic min/max checks on fee, m_feerate must be set if true bool fOverrideFeeRate; //! Override the wallet's m_pay_tx_fee if set - boost::optional m_feerate; + Optional m_feerate; //! Override the default confirmation target if set - boost::optional m_confirm_target; + Optional m_confirm_target; //! Override the wallet's m_signal_rbf if set - boost::optional m_signal_bip125_rbf; + Optional m_signal_bip125_rbf; //! Avoid partial use of funds sent to a given address bool m_avoid_partial_spends; //! Fee estimation mode to control arguments to estimateSmartFee diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index b63c2bdda6..418329a6fa 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -6,6 +6,8 @@ #include #include +#include + // Descending order comparator struct { bool operator()(const OutputGroup& a, const OutputGroup& b) const @@ -216,11 +218,11 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector& group nValueRet = 0; // List of values less than target - boost::optional lowest_larger; + Optional lowest_larger; std::vector applicable_groups; CAmount nTotalLower = 0; - random_shuffle(groups.begin(), groups.end(), GetRandInt); + Shuffle(groups.begin(), groups.end(), FastRandomContext()); for (const OutputGroup& group : groups) { if (group.m_value == nTargetValue) { diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 35cc5dd8fd..694857d04a 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -804,7 +804,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) create_time = FormatISO8601DateTime(it->second.nCreateTime); } if(pwallet->GetCScript(scriptid, script)) { - file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time); + file << strprintf("%s %s script=1", HexStr(script), create_time); file << strprintf(" # addr=%s\n", address); } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 7533c2a370..ff0275267c 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,60 @@ #include #include +// This enumeration determines the order of the CSV file header columns +typedef enum +{ + TRANSACTION_CSV_FIELD_DATETIME_HUMAN_READABLE, + TRANSACTION_CSV_FIELD_ACCOUNT, + TRANSACTION_CSV_FIELD_ADDRESS, + TRANSACTION_CSV_FIELD_CATEGORY, + TRANSACTION_CSV_FIELD_AMOUNT, + TRANSACTION_CSV_FIELD_LABEL, + TRANSACTION_CSV_FIELD_VOUT, + TRANSACTION_CSV_FIELD_FEE, + TRANSACTION_CSV_FIELD_CONFIRMATION, + TRANSACTION_CSV_FIELD_GENERATED, + TRANSACTION_CSV_FIELD_BLOCKHASH, + TRANSACTION_CSV_FIELD_BLOCKINDEX, + TRANSACTION_CSV_FIELD_BLOCKTIME, + TRANSACTION_CSV_FIELD_TRUSTED, + TRANSACTION_CSV_FIELD_WALLETCONFLICTS, + TRANSACTION_CSV_FIELD_TXID, + TRANSACTION_CSV_FIELD_TIME, + TRANSACTION_CSV_FIELD_TIMERECEIVED, + TRANSACTION_CSV_FIELD_COMPUTETIME, + TRANSACTION_CSV_FIELD_BIP125_REPLACEABLE, + TRANSACTION_CSV_FIELD_ABANDONED, + TRANSACTION_CSV_FIELD_WATCHONLY, + TRANSACTION_CSV_FIELD_COUNT +} TRANSACTION_CSV_FIELDS; + +const std::map CSV_HEADERS = +{ + {TRANSACTION_CSV_FIELD_DATETIME_HUMAN_READABLE, "date"}, + {TRANSACTION_CSV_FIELD_ACCOUNT, "account"}, + {TRANSACTION_CSV_FIELD_ADDRESS, "address"}, + {TRANSACTION_CSV_FIELD_CATEGORY, "category"}, + {TRANSACTION_CSV_FIELD_AMOUNT, "amount"}, + {TRANSACTION_CSV_FIELD_LABEL, "label"}, + {TRANSACTION_CSV_FIELD_VOUT, "vout"}, + {TRANSACTION_CSV_FIELD_FEE, "fee"}, + {TRANSACTION_CSV_FIELD_CONFIRMATION, "confirmation"}, + {TRANSACTION_CSV_FIELD_GENERATED, "generated"}, + {TRANSACTION_CSV_FIELD_BLOCKHASH, "blockhash"}, + {TRANSACTION_CSV_FIELD_BLOCKINDEX, "blockindex"}, + {TRANSACTION_CSV_FIELD_BLOCKTIME, "blocktime"}, + {TRANSACTION_CSV_FIELD_TRUSTED, "trusted"}, + {TRANSACTION_CSV_FIELD_WALLETCONFLICTS, "conflicts"}, + {TRANSACTION_CSV_FIELD_TXID, "txid"}, + {TRANSACTION_CSV_FIELD_TIME, "time"}, + {TRANSACTION_CSV_FIELD_TIMERECEIVED, "timereceived"}, + {TRANSACTION_CSV_FIELD_COMPUTETIME, "computetime"}, + {TRANSACTION_CSV_FIELD_BIP125_REPLACEABLE, "bip125-replaceable"}, + {TRANSACTION_CSV_FIELD_ABANDONED, "abandoned"}, + {TRANSACTION_CSV_FIELD_WATCHONLY, "watchonly"}, +}; + std::string GetDestType(CTxDestination dest) { if (dest.type() == typeid(CNoDestination)) return "CNoDestination"; @@ -163,6 +218,61 @@ static void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) entry.pushKV(item.first, item.second); } +static void WalletTxToCsv(std::vector& csvRecord, const CWalletTx& wtx) +{ + int confirms = wtx.GetDepthInMainChain(); + csvRecord[TRANSACTION_CSV_FIELD_CONFIRMATION] = std::to_string(confirms); + if (wtx.IsCoinBase()) + csvRecord[TRANSACTION_CSV_FIELD_GENERATED] = "true"; + if (confirms > 0) + { + csvRecord[TRANSACTION_CSV_FIELD_BLOCKHASH] = wtx.hashBlock.GetHex(); + csvRecord[TRANSACTION_CSV_FIELD_BLOCKINDEX] = std::to_string(wtx.nIndex); + csvRecord[TRANSACTION_CSV_FIELD_BLOCKTIME] = std::to_string(LookupBlockIndex(wtx.hashBlock)->GetBlockTime()); + } + else + { + csvRecord[TRANSACTION_CSV_FIELD_TRUSTED] = wtx.IsTrusted() ? "true" : "false"; + } + + uint256 hash = wtx.GetHash(); + csvRecord[TRANSACTION_CSV_FIELD_TXID] = hash.GetHex(); + + std::string conflicts = ""; + for (const uint256& conflict : wtx.GetConflicts()) + { + conflicts += conflict.GetHex(); + conflicts += ","; + } + csvRecord[TRANSACTION_CSV_FIELD_WALLETCONFLICTS] = conflicts; + csvRecord[TRANSACTION_CSV_FIELD_TIME] = std::to_string(wtx.GetTxTime()); + csvRecord[TRANSACTION_CSV_FIELD_DATETIME_HUMAN_READABLE] = FormatISO8601DateTime(wtx.GetTxTime()); + csvRecord[TRANSACTION_CSV_FIELD_TIMERECEIVED] = std::to_string(wtx.nTimeReceived); + csvRecord[TRANSACTION_CSV_FIELD_COMPUTETIME] = std::to_string(wtx.nComputeTime); + + // Add opt-in RBF status + std::string rbfStatus = "no"; + if (confirms <= 0) { + LOCK(mempool.cs); + RBFTransactionState rbfState = IsRBFOptIn(*wtx.tx, mempool); + if (rbfState == RBFTransactionState::UNKNOWN) + rbfStatus = "unknown"; + else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) + rbfStatus = "yes"; + } + csvRecord[TRANSACTION_CSV_FIELD_BIP125_REPLACEABLE] = rbfStatus; + + /* + // TODO + for (const std::pair& item : wtx.mapValue) + { + entry.pushKV(item.first, item.second); + } + */ +} + + + static std::string LabelFromValue(const UniValue& value) { std::string label = value.get_str(); @@ -1637,7 +1747,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); result.pushKV("address", EncodeDestination(dest)); - result.pushKV("redeemScript", HexStr(inner.begin(), inner.end())); + result.pushKV("redeemScript", HexStr(inner)); return result; } @@ -2031,13 +2141,6 @@ static UniValue listreceivedbylabel(const JSONRPCRequest& request) return ListReceived(pwallet, request.params, true); } -static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) -{ - if (IsValidDestination(dest)) { - entry.pushKV("address", EncodeDestination(dest)); - } -} - /** * List transactions based on the given criteria. * @@ -2067,15 +2170,14 @@ static void ListTransactions(CWallet* const pwallet, const CWalletTx& wtx, const for (const COutputEntry& s : listSent) { UniValue entry(UniValue::VOBJ); - if (involvesWatchonly || (::IsMine(*pwallet, s.destination) & ISMINE_WATCH_ONLY)) { + if (involvesWatchonly || (::IsMine(*pwallet, s.destination) & ISMINE_WATCH_ONLY)) entry.pushKV("involvesWatchonly", true); - } - MaybePushAddress(entry, s.destination); + if (IsValidDestination(s.destination)) + entry.pushKV("address", EncodeDestination(s.destination)); entry.pushKV("category", "send"); entry.pushKV("amount", ValueFromAmount(-s.amount)); - if (pwallet->mapAddressBook.count(s.destination)) { + if (pwallet->mapAddressBook.count(s.destination)) entry.pushKV("label", pwallet->mapAddressBook[s.destination].name); - } entry.pushKV("vout", s.vout); entry.pushKV("fee", ValueFromAmount(-nFee)); if (fLong) @@ -2091,17 +2193,23 @@ static void ListTransactions(CWallet* const pwallet, const CWalletTx& wtx, const for (const COutputEntry& r : listReceived) { std::string account; - if (pwallet->mapAddressBook.count(r.destination)) { + if (pwallet->mapAddressBook.count(r.destination)) account = pwallet->mapAddressBook[r.destination].name; - } if (fAllAccounts || (account == strAccount)) { UniValue entry(UniValue::VOBJ); - if (involvesWatchonly || (::IsMine(*pwallet, r.destination) & ISMINE_WATCH_ONLY)) { + if (involvesWatchonly || (::IsMine(*pwallet, r.destination) & ISMINE_WATCH_ONLY)) entry.pushKV("involvesWatchonly", true); + if (IsDeprecatedRPCEnabled("accounts")) + entry.pushKV("account", account); + if (IsValidDestination(r.destination)) + { + auto item = pwallet->mapAddressBook.find(r.destination); + if (item->first.type() == typeid(CKeyID)) + entry.pushKV("address", EncodeDestination(r.destination, false)); + else + entry.pushKV("address", EncodeDestination(r.destination)); } - if (IsDeprecatedRPCEnabled("accounts")) entry.pushKV("account", account); - MaybePushAddress(entry, r.destination); if (wtx.IsCoinBase()) { if (wtx.GetDepthInMainChain() < 1) @@ -2116,9 +2224,8 @@ static void ListTransactions(CWallet* const pwallet, const CWalletTx& wtx, const entry.pushKV("category", "receive"); } entry.pushKV("amount", ValueFromAmount(r.amount)); - if (pwallet->mapAddressBook.count(r.destination)) { + if (pwallet->mapAddressBook.count(r.destination)) entry.pushKV("label", account); - } entry.pushKV("vout", r.vout); if (fLong) WalletTxToJSON(wtx, entry); @@ -2284,7 +2391,7 @@ UniValue listtransactions(const JSONRPCRequest& request) { CWalletTx *const pwtx = (*it).second; if (pwtx != nullptr) - ListTransactions(pwallet, *pwtx, "", 0, true, retReversed, filter); + ListTransactions(pwallet, *pwtx, strAccount, 0, true, retReversed, filter); if ((int)retReversed.size() >= nCount + nFrom) break; @@ -2317,6 +2424,336 @@ UniValue listtransactions(const JSONRPCRequest& request) return retReversed; } +static void String_Tokenize(std::string const &str, const char delim, std::vector &out) +{ + size_t start; + size_t end = 0; + + while ((start = str.find_first_not_of(delim, end)) != std::string::npos) + { + end = str.find(delim, start); + out.push_back(str.substr(start, end - start)); + } +} + +static bool IsExportableTransaction(std::vector& record, std::vector& transactionFilters) +{ + bool exportable = transactionFilters.empty(); + for (uint16_t i = 0; !exportable && i < transactionFilters.size(); ++i) + { + exportable = std::find(record.begin(), record.end(), transactionFilters[i]) != record.end(); + } + + return exportable; +} + + +/** + * Export transactions based on the given criteria. + * + * @param pwallet The wallet. + * @param wtx The wallet transaction. + * @param strAccount The account, if any, or "*" for all. + * @param csvWrite The minimum confirmation depth. + * @param fLong Whether to include the details of the transaction. + * @param transactionFilters List of transaction filters + * @param ismineFilter The "is mine" filter bool. + */ +static void ExportTransactions(CWallet* const pwallet, const CWalletTx& wtx, const std::string& strAccount, + jay::util::CSVwrite& csv_write, bool fLong, std::vector& transactionFilters, const isminefilter& ismineFilter) +{ + CAmount nFee; + std::string dummy_account; + std::list listReceived; + std::list listSent; + + wtx.GetAmounts(listReceived, listSent, nFee, ismineFilter); + + bool fAllAccounts = (strAccount == std::string("*")); + bool involvesWatchonly = wtx.IsFromMe(ISMINE_WATCH_ONLY); + + // Sent + if ((!listSent.empty() || nFee != 0)) + { + for (auto s = listSent.rbegin(); s != listSent.rend(); ++s) + { + std::vector csvRecord(TRANSACTION_CSV_FIELD_COUNT); + if (involvesWatchonly || (::IsMine(*pwallet, s->destination) & ISMINE_WATCH_ONLY)) + csvRecord[TRANSACTION_CSV_FIELD_WATCHONLY] = "true"; + if ( IsValidDestination(s->destination) ) + csvRecord[TRANSACTION_CSV_FIELD_ADDRESS] = EncodeDestination(s->destination); + csvRecord[TRANSACTION_CSV_FIELD_CATEGORY] = "send"; + csvRecord[TRANSACTION_CSV_FIELD_AMOUNT] = ValueFromAmount(-s->amount).getValStr(); + if (pwallet->mapAddressBook.count(s->destination)) + csvRecord[TRANSACTION_CSV_FIELD_LABEL] = pwallet->mapAddressBook[s->destination].name; + csvRecord[TRANSACTION_CSV_FIELD_VOUT] = std::to_string(s->vout); + csvRecord[TRANSACTION_CSV_FIELD_FEE] = ValueFromAmount(-nFee).getValStr(); + WalletTxToCsv(csvRecord, wtx); + if (wtx.isAbandoned()) + csvRecord[TRANSACTION_CSV_FIELD_ABANDONED] = true; + if (IsExportableTransaction(csvRecord, transactionFilters)) + csv_write.WriteRecord(csvRecord, true); + } + } + + // Received + if (listReceived.size() > 0 /*&& wtx.GetDepthInMainChain() >= nMinDepth*/) + { + for (auto r = listReceived.rbegin(); r != listReceived.rend(); ++r) + { + std::string account; + if (pwallet->mapAddressBook.count(r->destination)) + account = pwallet->mapAddressBook[r->destination].name; + if (fAllAccounts || (account == strAccount)) + { + std::vector csvRecord(TRANSACTION_CSV_FIELD_COUNT); + if (involvesWatchonly || (::IsMine(*pwallet, r->destination) & ISMINE_WATCH_ONLY)) + csvRecord[TRANSACTION_CSV_FIELD_WATCHONLY] = "true"; + if(IsDeprecatedRPCEnabled("accounts")) + csvRecord[TRANSACTION_CSV_FIELD_ACCOUNT] = account; + if ( IsValidDestination(r->destination) ) + { + auto item = pwallet->mapAddressBook.find(r->destination); + if (item->first.type() == typeid(CKeyID)) + csvRecord[TRANSACTION_CSV_FIELD_ADDRESS] = EncodeDestination(r->destination, false); + else + csvRecord[TRANSACTION_CSV_FIELD_ADDRESS] = EncodeDestination(r->destination); + } + if (wtx.IsCoinBase()) + { + if (wtx.GetDepthInMainChain() < 1) + csvRecord[TRANSACTION_CSV_FIELD_CATEGORY] = "orphan"; + else if (wtx.GetBlocksToMaturity() > 0) + csvRecord[TRANSACTION_CSV_FIELD_CATEGORY] = "immature"; + else + csvRecord[TRANSACTION_CSV_FIELD_CATEGORY] = "generate"; + } + else + { + csvRecord[TRANSACTION_CSV_FIELD_CATEGORY] = "receive"; + } + csvRecord[TRANSACTION_CSV_FIELD_AMOUNT] = ValueFromAmount(r->amount).getValStr(); + if (pwallet->mapAddressBook.count(r->destination)) + csvRecord[TRANSACTION_CSV_FIELD_LABEL] = account; + csvRecord[TRANSACTION_CSV_FIELD_VOUT] = std::to_string(r->vout); + WalletTxToCsv(csvRecord, wtx); + if (IsExportableTransaction(csvRecord, transactionFilters)) { + csv_write.WriteRecord(csvRecord, true); + + // Basecoin sent transactions are nested here + // This section currently assumes that for the sent transaction: + // 1. The "data" type (transaction fee) is the first record + // 2. Each basecoin record denotes a subtotal of the entire amount sent + // If these assumptions are untrue then te code needs to be reevaluated + std::vector csvSubRecord(TRANSACTION_CSV_FIELD_COUNT); + for (unsigned int i = 0; i < wtx.tx->vpout.size(); ++i) { + auto pout = wtx.tx->vpout[i]; + bool fIsMyOutput = pwallet->IsMine(pout.get()); + + switch(pout->GetType()) + { + case OUTPUT_DATA: + { + // This typically denotes the transaction fee and is the first record + CTxOutData* outData = (CTxOutData*)pout.get(); + CAmount nFeeData; + if (outData->GetCTFee(nFeeData)) { + csvSubRecord[TRANSACTION_CSV_FIELD_FEE] = FormatMoney(nFeeData); + } + break; + } + + case OUTPUT_STANDARD: + if (!fIsMyOutput) { + if (pout->IsZerocoinMint()) { + // TODO? + } else { // basecoin + CTxDestination dest; + if (ExtractDestination(*pout->GetPScriptPubKey(), dest)) { + csvSubRecord[TRANSACTION_CSV_FIELD_CATEGORY] = "send"; + csvSubRecord[TRANSACTION_CSV_FIELD_ADDRESS] = EncodeDestination(dest, true); + csvSubRecord[TRANSACTION_CSV_FIELD_AMOUNT] = FormatMoney(pout->GetValue()); + WalletTxToCsv(csvSubRecord, wtx); + csv_write.WriteRecord(csvSubRecord, true); + } + } + } + break; + + case OUTPUT_CT: + // TODO? + break; + + case OUTPUT_RINGCT: + // TODO? + break; + + default: + break; + } // switch + } // for + } // if + } + } + } +} + + +static UniValue exporttransactions(const JSONRPCRequest& request) +{ + std::shared_ptr const wallet = GetWalletForJSONRPCRequest(request); + CWallet* const pwallet = wallet.get(); + + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + std::string help_text {}; + if (!IsDeprecatedRPCEnabled("accounts")) { + help_text = "exporttransactions (dummy filename start end categories include_watchonly)\n" + "\nExports transactions between 'start' date and 'end' date matching 'categories' to a CSV file.\n" + "Note: To export from a specified the \"account\", use this RPC with an \"account\" argument and restart\n" + "veild with -deprecatedrpc=accounts\n" + "\nArguments:\n" + "1. \"dummy\" (string, optional) If set, should be \"*\" for backwards compatibility.\n" + "2. \"filename\" (string, optional) The filename with path (either absolute or relative to veild) [default=/export/transactions.csv].\n" + "3. \"start\" (string, optional) The start date in the format YYYY-MM-DD [default=beginning of ISO8601 time].\n" + "4. \"end\" (string, optional) The end date in the format YYYY-MM-DD\n [default=present time]." + "5. \"filter\" (string, optional) A pipe(|) separated transaction filter [default=no filter]\n" + " Allowable filter words TBD\n" + "6. include_watchonly (bool, optional, default=false) Include transactions to watch-only addresses (see 'importaddress')\n" + "\nResult:\n" + "[\n" + " {\n" + " \"filename\" : { (string) The output filename with full absolute path\n" + " }\n" + "]\n" + + "\nExamples:\n" + "\nList all transactions\n" + + HelpExampleCli("exporttransactions", "") + + "\nList transactions over a date range\n" + + HelpExampleCli("exporttransactions", "\"*\" \"\" 2020-01-01 2020-12-31") + + "\nAs a json rpc call\n" + + HelpExampleRpc("exporttransactions", "\"*\", \"\", 2020-01-01, 2020-12-31"); + } else { + help_text = "exporttransactions ( \"account\" filename start end categories include_watchonly)\n" + "\nExports transactions between 'start' date and 'end' date matching 'categories' to a CSV file for 'account'.\n" + "\nArguments:\n" + "1. \"dummy\" (string, optional) If set, should be \"*\" for backwards compatibility.\n" + "2. \"filename\" (string, optional) The filename with path (either absolute or relative to veild) [default=/export/transactions.csv].\n" + "3. \"start\" (string, optional) The start date in the format YYYY-MM-DD [default=beginning of ISO8601 time].\n" + "4. \"end\" (string, optional) The end date in the format YYYY-MM-DD\n [default=present time]." + "5. \"filter\" (string, optional) A pipe(|) separated transaction filter [default=no filter]\n" + " Allowable filter words TBD\n" + "6. include_watchonly (bool, optional, default=false) Include transactions to watch-only addresses (see 'importaddress')\n" + "\nResult:\n" + "[\n" + " {\n" + " \"filename\" : { (string) The output filename with full absolute path\n" + " }\n" + "]\n" + + "\nExamples:\n" + "\nList all transactions\n" + + HelpExampleCli("exporttransactions", "") + + "\nList transactions over a date range\n" + + HelpExampleCli("exporttransactions", "\"*\" \"\" 2020-01-01 2020-12-31") + + "\nAs a json rpc call\n" + + HelpExampleRpc("exporttransactions", "\"*\", \"\", 2020-01-01, 2020-12-31"); + } + if (request.fHelp || request.params.size() > 6) + throw std::runtime_error(help_text); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + + // Determine export account + std::string strAccount = "*"; + if (!request.params[0].isNull()) { + strAccount = request.params[0].get_str(); + if (!IsDeprecatedRPCEnabled("accounts") && strAccount != "*") { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Dummy value must be set to \"*\""); + } + } + + // Determine export path + const fs::path DEFAULT_EXPORT_DIR = GetDataDir() / "export"; + const fs::path DEFAULT_EXPORT_PATH = DEFAULT_EXPORT_DIR / "transactions.csv"; + std::string exportPath = !request.params[1].isNull() && !request.params[1].get_str().empty() + ? request.params[1].get_str() : DEFAULT_EXPORT_PATH.string(); + fs::path exportDir = fs::path(exportPath).parent_path(); + if (!exportDir.string().empty() && !fs::exists(exportDir)) { + if (!fs::create_directories(exportDir)) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot create export directory " + exportDir.string()); + } + + // Determine export start and end dates + std::string startDateString = !request.params[2].isNull() ? request.params[2].get_str() : ""; + std::string endDateString = !request.params[3].isNull() ? request.params[3].get_str() : ""; + if (startDateString != "") + startDateString += ":00:00"; + if (endDateString != "") + endDateString += ":00:00"; + if (startDateString != "" && !ISO8601Date_Validate(startDateString)) + throw JSONRPCError(RPC_INVALID_PARAMETER, "start date format is invalid:" + startDateString); + if (endDateString != "" && !ISO8601Date_Validate(endDateString)) + throw JSONRPCError(RPC_INVALID_PARAMETER, "end date format is invalid:" + endDateString); + uint64_t startDate = 0; + uint64_t endDate = ISO8601Date_Now(); + if (startDateString != "") + startDate = ISO8601Date_FromString(startDateString); + if (endDateString != "") + endDate = ISO8601Date_FromString(endDateString); + + // Prepare transaction filter + std::string transactionFilterString = !request.params[4].isNull() ? request.params[4].get_str() : ""; + vector transactionFilters; + String_Tokenize(transactionFilterString, '|', transactionFilters); + + // Prepare watchonly filter + isminefilter ismineFilter = ISMINE_SPENDABLE; + if(!request.params[5].isNull()) + if(request.params[5].get_bool()) + ismineFilter = ismineFilter | ISMINE_WATCH_ONLY; + + // Create CSV file + jay::util::CSVwrite csv_write; + if (!csv_write.Open( exportPath, jay::util::CSVwrite::Flags::truncate )) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open transactions export file " + exportPath); + for (uint8_t field = 0; field < TRANSACTION_CSV_FIELD_COUNT; ++field) + { + csv_write.WriteField(CSV_HEADERS.at((TRANSACTION_CSV_FIELDS)field), false); + } + csv_write.WriteTerminator(); + + { + LOCK2(cs_main, pwallet->cs_wallet); + const CWallet::TxItems &txOrdered = pwallet->wtxOrdered; + + for (CWallet::TxItems::const_iterator it = txOrdered.begin(); it != txOrdered.end(); ++it) + { + CWalletTx *const pwtx = (*it).second; + if (pwtx == nullptr) + continue; + if (pwtx->GetTxTime() < startDate) + continue; + if (pwtx->GetTxTime() > endDate) + break; + ExportTransactions(pwallet, *pwtx, strAccount, csv_write, true, transactionFilters, ismineFilter); + } + } + + if (!csv_write.Close()) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot close transactions export file " + exportPath); + + UniValue reply(UniValue::VOBJ); + reply.pushKV("filename", exportPath); + + return reply; +} + static UniValue listsinceblock(const JSONRPCRequest& request) { std::shared_ptr const wallet = GetWalletForJSONRPCRequest(request); @@ -3925,7 +4362,7 @@ static UniValue listunspent(const JSONRPCRequest& request) } } - entry.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + entry.pushKV("scriptPubKey", HexStr(scriptPubKey)); // If RingCT, get the ephemeral pubkey if (outType == OUTPUT_RINGCT) { @@ -4643,7 +5080,7 @@ class DescribeWalletAddressVisitor : public boost::static_visitor std::vector> solutions_data; Solver(subscript, which_type, solutions_data); obj.pushKV("script", GetTxnOutputType(which_type)); - obj.pushKV("hex", HexStr(subscript.begin(), subscript.end())); + obj.pushKV("hex", HexStr(subscript)); CTxDestination embedded; UniValue a(UniValue::VARR); @@ -4655,7 +5092,7 @@ class DescribeWalletAddressVisitor : public boost::static_visitor UniValue wallet_detail = boost::apply_visitor(*this, embedded); subobj.pushKVs(wallet_detail); subobj.pushKV("address", EncodeDestination(embedded)); - subobj.pushKV("scriptPubKey", HexStr(subscript.begin(), subscript.end())); + subobj.pushKV("scriptPubKey", HexStr(subscript)); // Always report the pubkey at the top level, so that `getnewaddress()['pubkey']` always works. if (subobj.exists("pubkey")) obj.pushKV("pubkey", subobj["pubkey"]); obj.pushKV("embedded", std::move(subobj)); @@ -4668,7 +5105,7 @@ class DescribeWalletAddressVisitor : public boost::static_visitor for (size_t i = 1; i < solutions_data.size() - 1; ++i) { CPubKey key(solutions_data[i].begin(), solutions_data[i].end()); if (include_addresses) a.push_back(EncodeDestination(key.GetID())); - pubkeys.push_back(HexStr(key.begin(), key.end())); + pubkeys.push_back(HexStr(key)); } obj.pushKV("pubkeys", std::move(pubkeys)); } @@ -4879,7 +5316,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ret.pushKV("address", currentAddress); CScript scriptPubKey = GetScriptForDestination(dest); - ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); //isminetype mine = IsMine(*pwallet, dest); isminetype mine = pwallet->IsMine(dest); @@ -5563,6 +6000,7 @@ static const CRPCCommand commands[] = { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, + { "wallet", "exporttransactions", &exporttransactions, {"account", "filename", "start", "end", "transactions"} }, { "wallet", "getaddressinfo", &getaddressinfo, {"address"} }, { "wallet", "getbalance", &getbalance, {"account|dummy","minconf","include_watchonly"} }, { "wallet", "getspendablebalance", &getspendablebalance, {} }, diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 6a70584f46..6b10b40670 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Get the final tx CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - std::string final_hex = HexStr(ssTx.begin(), ssTx.end()); + std::string final_hex = HexStr(ssTx); BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); } diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 35279163b6..1c26fabfb6 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -24,10 +24,10 @@ static void TestPassphraseSingle(const std::vector& vchSalt, cons if(!correctKey.empty()) BOOST_CHECK_MESSAGE(memcmp(crypt.vchKey.data(), correctKey.data(), crypt.vchKey.size()) == 0, \ - HexStr(crypt.vchKey.begin(), crypt.vchKey.end()) + std::string(" != ") + HexStr(correctKey.begin(), correctKey.end())); + HexStr(crypt.vchKey) + std::string(" != ") + HexStr(correctKey)); if(!correctIV.empty()) BOOST_CHECK_MESSAGE(memcmp(crypt.vchIV.data(), correctIV.data(), crypt.vchIV.size()) == 0, - HexStr(crypt.vchIV.begin(), crypt.vchIV.end()) + std::string(" != ") + HexStr(correctIV.begin(), correctIV.end())); + HexStr(crypt.vchIV) + std::string(" != ") + HexStr(correctIV)); } static void TestPassphrase(const std::vector& vchSalt, const SecureString& passphrase, uint32_t rounds, diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 1fdb177429..5f6f2b9ec2 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3240,7 +3240,7 @@ bool CWallet::SelectCoins(const std::vector& vAvailableCoins, const CAm // Cases where we have 11+ outputs all pointing to the same destination may result in // privacy leaks as they will potentially be deterministically sorted. We solve that by // explicitly shuffling the outputs before processing - std::shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); + Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } std::vector groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends); @@ -3772,7 +3772,7 @@ bool CWallet::CreateTransaction(const std::vector& vecSend, CTransac // Shuffle selected coins and fill in final vin txNew.vin.clear(); std::vector selected_coins(setCoins.begin(), setCoins.end()); - std::shuffle(selected_coins.begin(), selected_coins.end(), FastRandomContext()); + Shuffle(selected_coins.begin(), selected_coins.end(), FastRandomContext()); // Note how the sequence number is set to non-maxint so that // the nLockTime set above actually works.