Merge pull request #891

Support type relaxation for JSON import

CHANGELOG.md

@@ -11,6 +11,10 @@ Every entry has a category for which we use the following visual abbreviations:
 
 ## Unreleased
 
+- 🎁 The `import json` command's type restrictions are more relaxed now, and can
+  additionally convert from JSON strings to VAST internal data types.
+  [#891](https://github.com/tenzir/vast/pull/891)
+
 - 🎁 VAST now supports /etc/vast/vast.conf as an additional fallback for the
   configuration file. The following file locations are looked at in order: Path
   specified on the command line via `--config=path/to/vast.conf`, `vast.conf` in

libvast/src/format/json.cpp

@@ -136,6 +136,51 @@ struct convert {
     return xs;
   }
 
+  caf::expected<data>
+  operator()(const json::string& str, const bool_type&) const {
+    if (bool x; parsers::json_boolean(str, x))
+      return x;
+    return make_error(ec::convert_error, "cannot convert from", str, "to bool");
+  }
+
+  caf::expected<data>
+  operator()(const json::string& str, const real_type&) const {
+    if (real x; parsers::json_number(str, x))
+      return x;
+    return make_error(ec::convert_error, "cannot convert from", str, "to real");
+  }
+
+  caf::expected<data>
+  operator()(const json::string& str, const integer_type&) const {
+    if (integer x; parsers::json_int(str, x))
+      return x;
+    if (real x; parsers::json_number(str, x)) {
+      VAST_WARNING_ANON("json-reader narrowed", str, "to type int");
+      return detail::narrow_cast<integer>(x);
+    }
+    return make_error(ec::convert_error, "cannot convert from", str, "to int");
+  }
+
+  caf::expected<data>
+  operator()(const json::string& str, const count_type&) const {
+    if (count x; parsers::json_count(str, x))
+      return x;
+    if (real x; parsers::json_number(str, x)) {
+      VAST_WARNING_ANON("json-reader narrowed", str, "to type count");
+      return detail::narrow_cast<count>(x);
+    }
+    return make_error(ec::convert_error, "cannot convert from", str,
+                      "to count");
+  }
+
+  caf::expected<data> operator()(json::string str, const port_type&) const {
+    if (port x; parsers::port(str, x))
+      return x;
+    if (port::number_type x; parsers::u16(str, x))
+      return port{x};
+    return make_error(ec::convert_error, "cannot convert from", str, "to port");
+  }
+
   template <class T, class U>
   caf::expected<data> operator()(T, U) const {
     VAST_ERROR_ANON("json-reader cannot convert from",

libvast/test/format/json.cpp

@@ -131,4 +131,19 @@ TEST_DISABLED(suricata) {
   CHECK(slices[0]->at(0, 19) == data{count{4520}});
 }
 
+TEST(json hex number parser) {
+  using namespace parsers;
+  double x;
+  CHECK(json_number("123.0", x));
+  CHECK_EQUAL(x, 123.0);
+  CHECK(json_number("-123.0", x));
+  CHECK_EQUAL(x, -123.0);
+  CHECK(json_number("123", x));
+  CHECK_EQUAL(x, 123.0);
+  CHECK(json_number("+123", x));
+  CHECK_EQUAL(x, 123.0);
+  CHECK(json_number("0xFF", x));
+  CHECK_EQUAL(x, 255.0);
+}
+
 FIXTURE_SCOPE_END()

libvast/test/parseable.cpp

@@ -501,6 +501,26 @@ TEST(unsigned integral) {
   CHECK_EQUAL(x, 12u);
 }
 
+TEST(unsigned hexadecimal integral) {
+  using namespace parsers;
+  auto p = ignore(-hex_prefix) >> hex64;
+  unsigned x;
+  CHECK(p("1234", x));
+  CHECK_EQUAL(x, 0x1234);
+  CHECK(p("13BFC3d1", x));
+  CHECK_EQUAL(x, 0x13BFC3d1);
+  CHECK(p("FF", x));
+  CHECK_EQUAL(x, 0xFF);
+  CHECK(p("ff00", x));
+  CHECK_EQUAL(x, 0xff00);
+  CHECK(p("0X12ab", x));
+  CHECK_EQUAL(x, 0X12ab);
+  CHECK(p("0x3e7", x));
+  CHECK_EQUAL(x, 0x3e7);
+  CHECK(p("0x0000aa", x));
+  CHECK_EQUAL(x, 0x0000aa);
+}
+
 TEST(signed integral with digit constraints) {
   constexpr auto max = 4;
   constexpr auto min = 2;
@@ -587,7 +607,7 @@ TEST(byte) {
   auto f = str.begin();
   auto l = f + 1;
   auto u8 = uint8_t{0};
-  CHECK(byte(f, l, u8));
+  CHECK(parsers::byte(f, l, u8));
   CHECK(u8 == 0x01u);
   CHECK(f == l);
   MESSAGE("big endian");

libvast/vast/concept/parseable/numeric/integral.hpp

@@ -13,9 +13,10 @@
 
 #pragma once
 
-#include <cstdint>
+#include "vast/concept/parseable/core.hpp"
+#include "vast/detail/coding.hpp"
 
-#include "vast/concept/parseable/core/parser.hpp"
+#include <cstdint>
 
 namespace vast {
 namespace detail {
@@ -32,32 +33,44 @@ bool parse_sign(Iterator& i) {
 
 } // namespace detail
 
-template <
-  class T,
-  int MaxDigits = std::numeric_limits<T>::digits10 + 1,
-  int MinDigits = 1,
-  int Radix = 10
->
+template <class T,
+          // Note that it is fine to have this specific for base 10, as other
+          // bases must be specified as a later template parameters anyways.
+          int MaxDigits = std::numeric_limits<T>::digits10 + 1,
+          int MinDigits = 1, int Radix = 10>
 struct integral_parser
   : parser<integral_parser<T, MaxDigits, MinDigits, Radix>> {
-  static_assert(Radix == 10, "unsupported radix");
+  static_assert(Radix == 10 || Radix == 16, "unsupported radix");
   static_assert(MinDigits > 0, "need at least one minimum digit");
   static_assert(MaxDigits > 0, "need at least one maximum digit");
   static_assert(MinDigits <= MaxDigits, "maximum cannot exceed minimum");
 
   using attribute = T;
 
   static bool isdigit(char c) {
-    return c >= '0' && c <= '9';
+    if constexpr (Radix == 10)
+      return c >= '0' && c <= '9';
+    else if constexpr (Radix == 16)
+      return std::isxdigit(c);
+    else
+      static_assert(detail::always_false_v<decltype(Radix)>, "unsupported "
+                                                             "radix");
   }
 
   template <class Iterator, class Attribute, class F>
   static auto accumulate(Iterator& f, const Iterator& l, Attribute& a, F acc) {
     if (f == l)
       return false;
     int digits = 0;
-    for (a = 0; isdigit(*f) && f != l && digits < MaxDigits; ++f, ++digits)
-      acc(a, *f - '0');
+    for (a = 0; isdigit(*f) && f != l && digits < MaxDigits; ++f, ++digits) {
+      if constexpr (Radix == 10)
+        acc(a, *f - '0');
+      else if constexpr (Radix == 16)
+        acc(a, detail::hex_to_byte(*f));
+      else
+        static_assert(detail::always_false_v<decltype(Radix)>, "unsupported "
+                                                               "radix");
+    }
     return digits >= MinDigits;
   }
 
@@ -141,6 +154,12 @@ auto const u16 = integral_parser<uint16_t>{};
 auto const u32 = integral_parser<uint32_t>{};
 auto const u64 = integral_parser<uint64_t>{};
 
+auto const hex_prefix = ignore(lit{"0x"} | lit{"0X"});
+auto const hex8 = integral_parser<uint8_t, 2, 1, 16>{};
+auto const hex16 = integral_parser<uint16_t, 4, 1, 16>{};
+auto const hex32 = integral_parser<uint32_t, 8, 1, 16>{};
+auto const hex64 = integral_parser<uint64_t, 16, 1, 16>{};
+
 } // namespace parsers
 } // namespace vast
 

libvast/vast/concept/parseable/vast/json.hpp

@@ -13,17 +13,36 @@
 
 #pragma once
 
-#include "vast/json.hpp"
-
 #include "vast/concept/parseable/core.hpp"
 #include "vast/concept/parseable/core/rule.hpp"
 #include "vast/concept/parseable/numeric/bool.hpp"
+#include "vast/concept/parseable/numeric/integral.hpp"
 #include "vast/concept/parseable/numeric/real.hpp"
 #include "vast/concept/parseable/string/char_class.hpp"
 #include "vast/concept/parseable/string/quoted_string.hpp"
+#include "vast/detail/narrow.hpp"
+#include "vast/json.hpp"
 
 namespace vast {
 
+namespace parsers {
+
+// clang-format off
+static auto const json_boolean
+  = parsers::boolean;
+static auto const json_int
+  = parsers::i64;
+static auto const json_count
+  = (parsers::hex_prefix >> parsers::hex64)
+  | parsers::u64;
+static auto const json_number
+  = (parsers::hex_prefix >> parsers::hex64 ->* [](uint64_t x) {
+        return detail::narrow_cast<json::number>(x); })
+  | parsers::real_opt_dot;
+// clang-format on
+
+} // namespace parsers
+
 struct json_parser : parser<json_parser> {
   using attribute = json;
 
@@ -39,15 +58,21 @@ struct json_parser : parser<json_parser> {
     auto rbrace = ws >> '}' >> ws;
     auto delim = ws >> ',' >> ws;
     // clang-format off
-    auto null = ws >> "null"_p ->* [] { return json::null{}; };
+    auto null = "null"_p ->* [] { return json::null{}; };
     // clang-format on
-    auto boolean = ws >> parsers::boolean;
-    auto string = ws >> parsers::qqstr;
-    auto number = ws >> parsers::real_opt_dot;
+    auto string = parsers::qqstr;
     auto array = as<json::array>(lbracket >> ~(ref(j) % delim) >> rbracket);
-    auto key_value = ws >> string >> ws >> ':' >> ws >> ref(j);
+    auto key_value = string >> ws >> ':' >> ws >> ref(j);
     auto object = as<json::object>(lbrace >> ~(key_value % delim) >> rbrace);
-    j = null | boolean | number | string | array | object;
+    // clang-format off
+    j = ws >> ( null
+              | json_boolean
+              | json_number
+              | string
+              | array
+              | object
+              );
+    // clang-format on
     return j(f, l, x);
   }
 };

libvast/vast/format/json.hpp

@@ -119,7 +119,7 @@ class reader final : public multi_layout_reader {
   /// @param options Additional options.
   /// @param in The stream of JSON objects.
   explicit reader(caf::atom_value table_slice_type,
-                  const caf::settings& options,
+                  const caf::settings& /*options*/,
                   std::unique_ptr<std::istream> in = nullptr);
 
   void reset(std::unique_ptr<std::istream> in);
@@ -153,7 +153,7 @@ class reader final : public multi_layout_reader {
 
 template <class Selector>
 reader<Selector>::reader(caf::atom_value table_slice_type,
-                         const caf::settings& /*options*/,
+                         const caf::settings& options,
                          std::unique_ptr<std::istream> in)
   : super(table_slice_type) {
   if (in != nullptr)