From 2a8932ecfd3d475c91359ca3ed79e4c2e570ce02 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Fri, 29 May 2020 14:06:54 +0200
Subject: [PATCH 01/10] Add toggle for strict/relaxed JSON import mode

---
 libvast/src/format/json.cpp        | 11 ++++++++---
 libvast/src/system/application.cpp |  6 +++---
 libvast/test/format/json.cpp       |  2 +-
 libvast/vast/format/json.hpp       | 11 ++++++++---
 vast.conf                          |  5 ++++-
 5 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp
index 238e026f28b..5dc64a1ccbf 100644
--- a/libvast/src/format/json.cpp
+++ b/libvast/src/format/json.cpp
@@ -37,7 +37,7 @@
 namespace vast::format::json {
 namespace {
 
-struct convert {
+struct strict_convert {
   template <class T>
   using expected = caf::expected<T>;
   using json = vast::json;
@@ -145,6 +145,10 @@ struct convert {
   }
 };
 
+struct relaxed_convert : strict_convert {
+  using strict_convert::operator();
+};
+
 const vast::json* lookup(std::string_view field, const vast::json::object& xs) {
   VAST_ASSERT(!field.empty());
   auto lookup_flat = [&]() {
@@ -179,7 +183,7 @@ const char* writer::name() const {
 }
 
 caf::error add(table_slice_builder& builder, const vast::json::object& xs,
-               const record_type& layout) {
+               const record_type& layout, bool strict) {
   for (auto& field : layout.fields) {
     auto i = lookup(field.name, xs);
     // Non-existing fields are treated as empty (unset).
@@ -189,7 +193,8 @@ caf::error add(table_slice_builder& builder, const vast::json::object& xs,
                                            "slice builder");
       continue;
     }
-    auto x = caf::visit(convert{}, *i, field.type);
+    auto x = strict ? caf::visit(strict_convert{}, *i, field.type)
+                    : caf::visit(relaxed_convert{}, *i, field.type);
     if (!x)
       return make_error(ec::convert_error, x.error().context(),
                         "could not convert", field.name, ":", to_string(*i));
diff --git a/libvast/src/system/application.cpp b/libvast/src/system/application.cpp
index ec531e68a15..e59dce4ccbe 100644
--- a/libvast/src/system/application.cpp
+++ b/libvast/src/system/application.cpp
@@ -177,9 +177,9 @@ auto make_import_command() {
   import_->add_subcommand("csv", "imports CSV logs from STDIN or file",
                           documentation::vast_import_csv,
                           source_opts("?import.csv"));
-  import_->add_subcommand("json", "imports JSON with schema",
-                          documentation::vast_import_json,
-                          source_opts("?import.json"));
+  import_->add_subcommand(
+    "json", "imports JSON with schema", documentation::vast_import_json,
+    source_opts("?import.json").add<bool>("strict", "strict type matching"));
   import_->add_subcommand("suricata", "imports suricata eve json",
                           documentation::vast_import_suricata,
                           source_opts("?import.suricata"));
diff --git a/libvast/test/format/json.cpp b/libvast/test/format/json.cpp
index 3fd3f2248f7..b23201c969f 100644
--- a/libvast/test/format/json.cpp
+++ b/libvast/test/format/json.cpp
@@ -104,7 +104,7 @@ TEST(json to data) {
   })json";
   auto jn = unbox(to<json>(str));
   auto xs = caf::get<json::object>(jn);
-  format::json::add(builder, xs, flat);
+  format::json::add(builder, xs, flat, true);
   auto ptr = builder.finish();
   REQUIRE(ptr);
   CHECK(ptr->at(0, 11) == data{enumeration{2}});
diff --git a/libvast/vast/format/json.hpp b/libvast/vast/format/json.hpp
index 08f324e85bd..42d0e37a885 100644
--- a/libvast/vast/format/json.hpp
+++ b/libvast/vast/format/json.hpp
@@ -31,6 +31,7 @@
 
 #include <caf/expected.hpp>
 #include <caf/fwd.hpp>
+#include <caf/settings.hpp>
 
 namespace vast::format::json {
 
@@ -51,9 +52,10 @@ class writer : public ostream_writer {
 /// @param builder The builder to add the JSON object to.
 /// @param xs The JSON object to add to *builder.
 /// @param layout The record type describing *xs*.
+/// @param strict Toggle between strict and relaxed type matching.
 /// @returns An error iff the operation failed.
 caf::error add(table_slice_builder& builder, const vast::json::object& xs,
-               const record_type& layout);
+               const record_type& layout, bool strict);
 
 /// @relates reader
 struct default_selector {
@@ -147,17 +149,20 @@ class reader final : public multi_layout_reader {
   mutable size_t num_invalid_lines_ = 0;
   mutable size_t num_unknown_layouts_ = 0;
   mutable size_t num_lines_ = 0;
+  bool strict_ = false;
 };
 
 // -- implementation ----------------------------------------------------------
 
 template <class Selector>
 reader<Selector>::reader(caf::atom_value table_slice_type,
-                         const caf::settings& /*options*/,
+                         const caf::settings& options,
                          std::unique_ptr<std::istream> in)
   : super(table_slice_type) {
   if (in != nullptr)
     reset(std::move(in));
+  std::string category = vast::defaults::import::json::category;
+  strict_ = caf::get_or(options, category + ".strict", false);
 }
 
 template <class Selector>
@@ -256,7 +261,7 @@ caf::error reader<Selector>::read_impl(size_t max_events, size_t max_slice_size,
     bptr = builder(*layout);
     if (bptr == nullptr)
       return make_error(ec::parse_error, "unable to get a builder");
-    if (auto err = add(*bptr, *xs, *layout)) {
+    if (auto err = add(*bptr, *xs, *layout, strict_)) {
       err.context() += caf::make_message("line", lines_->line_number());
       return finish(cons, err);
     }
diff --git a/vast.conf b/vast.conf
index ab5a4f1239f..09b01b841a9 100644
--- a/vast.conf
+++ b/vast.conf
@@ -77,7 +77,10 @@ export {
   ; The `vast export json` command exports events formatted as JSONL (line-
   ; delimited JSON).
   json {
-    ; For available options, see export.ascii.
+    ; Strict type matching.
+    ;strict = false
+
+    ; For additionally available options, see export.ascii.
   }
 
   ; The `vast export null` command exports events from a given query without

From 2870aa564de74855b2559c4b0225134d56430038 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Fri, 29 May 2020 14:18:38 +0200
Subject: [PATCH 02/10] Support JSON type relaxation for bool

---
 libvast/src/format/json.cpp                  |  6 ++++++
 libvast/vast/concept/parseable/vast/json.hpp |  9 +++++++--
 schema/sysmon.schema                         | 12 ++++++------
 scripts/generate-sysmon-schema.py            |  2 +-
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp
index 5dc64a1ccbf..9e2f634c645 100644
--- a/libvast/src/format/json.cpp
+++ b/libvast/src/format/json.cpp
@@ -147,6 +147,12 @@ struct strict_convert {
 
 struct relaxed_convert : strict_convert {
   using strict_convert::operator();
+
+  caf::expected<data> operator()(json::string str, const bool_type&) const {
+    if (bool res; parsers::json_boolean(str, res))
+      return res;
+    return make_error(ec::convert_error, "cannot convert from", str, "to bool");
+  }
 };
 
 const vast::json* lookup(std::string_view field, const vast::json::object& xs) {
diff --git a/libvast/vast/concept/parseable/vast/json.hpp b/libvast/vast/concept/parseable/vast/json.hpp
index 658c446b347..365829e841a 100644
--- a/libvast/vast/concept/parseable/vast/json.hpp
+++ b/libvast/vast/concept/parseable/vast/json.hpp
@@ -24,6 +24,12 @@
 
 namespace vast {
 
+namespace parsers {
+
+static auto const json_boolean = ignore(*parsers::space) >> parsers::boolean;
+
+} // namespace parsers
+
 struct json_parser : parser<json_parser> {
   using attribute = json;
 
@@ -41,13 +47,12 @@ struct json_parser : parser<json_parser> {
     // clang-format off
     auto null = ws >> "null"_p ->* [] { return json::null{}; };
     // clang-format on
-    auto boolean = ws >> parsers::boolean;
     auto string = ws >> parsers::qqstr;
     auto number = ws >> parsers::real_opt_dot;
     auto array = as<json::array>(lbracket >> ~(ref(j) % delim) >> rbracket);
     auto key_value = ws >> string >> ws >> ':' >> ws >> ref(j);
     auto object = as<json::object>(lbrace >> ~(key_value % delim) >> rbrace);
-    j = null | boolean | number | string | array | object;
+    j = null | json_boolean | number | string | array | object;
     return j(f, l, x);
   }
 };
diff --git a/schema/sysmon.schema b/schema/sysmon.schema
index 1bfdcf776c7..395361179df 100644
--- a/schema/sysmon.schema
+++ b/schema/sysmon.schema
@@ -367,7 +367,7 @@ type sysmon.FileDelete = record {
     // Hashes captured by sysmon driver of the deleted file
     Hashes: string,
     // TBD
-    IsExecutable: string,
+    IsExecutable: bool,
     // States if the file was archived when deleted
     Archived: string,
 }
@@ -392,9 +392,9 @@ type sysmon.NetworkConnection = record {
     // Protocol being used for the network connection
     Protocol: string,
     // Indicated process initiated tcp connection
-    Initiated: string,
+    Initiated: bool,
     // is the source ip an Ipv6
-    SourceIsIpv6: string,
+    SourceIsIpv6: bool,
     // source ip address that made the network connection
     SourceIp: addr,
     // name of the host that made the network connection
@@ -404,7 +404,7 @@ type sysmon.NetworkConnection = record {
     // name of the source port being used (i.e. netbios-dgm)
     SourcePortName: string,
     // is the destination ip an Ipv6
-    DestinationIsIpv6: string,
+    DestinationIsIpv6: bool,
     // ip address destination
     DestinationIp: addr,
     // name of the host that received the network connection
@@ -457,7 +457,7 @@ type sysmon.DriverLoaded = record {
     // Hashes captured by sysmon driver
     Hashes: string,
     // is the driver loaded signed
-    Signed: string,
+    Signed: bool,
     // The signer
     Signature: string,
     // status of the signature (i.e valid)
@@ -497,7 +497,7 @@ type sysmon.ImageLoaded = record {
     // hash is a full hash of the file with the algorithms in the HashType field
     Hashes: string,
     // is the image loaded signed
-    Signed: string,
+    Signed: bool,
     // The signer
     Signature: string,
     // status of the signature (i.e valid)
diff --git a/scripts/generate-sysmon-schema.py b/scripts/generate-sysmon-schema.py
index b952199c5cf..1c842921051 100755
--- a/scripts/generate-sysmon-schema.py
+++ b/scripts/generate-sysmon-schema.py
@@ -91,7 +91,7 @@ def map_type(name):
     elif name == "date":
         return "time #timestamp"
     elif name == "boolean" or name == "bool":
-        return "string" # sysmon sends some booleans as strings
+        return "bool"
     elif name == "ip":
         return "addr"
     else:

From 18d46fecd714d117151691a49ff7df154aeb9d26 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Fri, 29 May 2020 14:44:59 +0200
Subject: [PATCH 03/10] Support JSON type relaxation for number/int/count

---
 libvast/src/format/json.cpp                  | 27 +++++++++++
 libvast/vast/concept/parseable/vast/json.hpp | 21 ++++++--
 schema/sysmon.schema                         | 50 ++++++++++----------
 scripts/generate-sysmon-schema.py            |  2 +-
 4 files changed, 70 insertions(+), 30 deletions(-)

diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp
index 9e2f634c645..db42fb2413c 100644
--- a/libvast/src/format/json.cpp
+++ b/libvast/src/format/json.cpp
@@ -153,6 +153,33 @@ struct relaxed_convert : strict_convert {
       return res;
     return make_error(ec::convert_error, "cannot convert from", str, "to bool");
   }
+
+  caf::expected<data> operator()(json::string str, const real_type&) const {
+    if (real res; parsers::json_number(str, res))
+      return res;
+    return make_error(ec::convert_error, "cannot convert from", str, "to real");
+  }
+
+  caf::expected<data> operator()(json::string str, const integer_type&) const {
+    if (integer res; parsers::json_int(str, res))
+      return res;
+    if (real res; parsers::json_number(str, res)) {
+      VAST_WARNING_ANON("json-reader narrowed", str, "to type int");
+      return detail::narrow_cast<integer>(res);
+    }
+    return make_error(ec::convert_error, "cannot convert from", str, "to int");
+  }
+
+  caf::expected<data> operator()(json::string str, const count_type&) const {
+    if (count res; parsers::json_count(str, res))
+      return res;
+    if (real res; parsers::json_number(str, res)) {
+      VAST_WARNING_ANON("json-reader narrowed", str, "to type int");
+      return detail::narrow_cast<count>(res);
+    }
+    return make_error(ec::convert_error, "cannot convert from", str,
+                      "to count");
+  }
 };
 
 const vast::json* lookup(std::string_view field, const vast::json::object& xs) {
diff --git a/libvast/vast/concept/parseable/vast/json.hpp b/libvast/vast/concept/parseable/vast/json.hpp
index 365829e841a..18848293dea 100644
--- a/libvast/vast/concept/parseable/vast/json.hpp
+++ b/libvast/vast/concept/parseable/vast/json.hpp
@@ -13,20 +13,27 @@
 
 #pragma once
 
-#include "vast/json.hpp"
-
 #include "vast/concept/parseable/core.hpp"
 #include "vast/concept/parseable/core/rule.hpp"
 #include "vast/concept/parseable/numeric/bool.hpp"
+#include "vast/concept/parseable/numeric/integral.hpp"
 #include "vast/concept/parseable/numeric/real.hpp"
 #include "vast/concept/parseable/string/char_class.hpp"
 #include "vast/concept/parseable/string/quoted_string.hpp"
+#include "vast/json.hpp"
 
 namespace vast {
 
 namespace parsers {
 
 static auto const json_boolean = ignore(*parsers::space) >> parsers::boolean;
+static auto const json_number
+  = ignore(*parsers::space) >> parsers::real_opt_dot;
+
+// These parsers are only needed for relaxed conversion from JSON string to VAST
+// type, they are not part of the actual JSON specification.
+static auto const json_int = ignore(*parsers::space) >> parsers::i64;
+static auto const json_count = ignore(*parsers::space) >> parsers::u64;
 
 } // namespace parsers
 
@@ -48,11 +55,17 @@ struct json_parser : parser<json_parser> {
     auto null = ws >> "null"_p ->* [] { return json::null{}; };
     // clang-format on
     auto string = ws >> parsers::qqstr;
-    auto number = ws >> parsers::real_opt_dot;
     auto array = as<json::array>(lbracket >> ~(ref(j) % delim) >> rbracket);
     auto key_value = ws >> string >> ws >> ':' >> ws >> ref(j);
     auto object = as<json::object>(lbrace >> ~(key_value % delim) >> rbrace);
-    j = null | json_boolean | number | string | array | object;
+    // clang-format off
+    j = null
+      | json_boolean
+      | json_number
+      | string
+      | array
+      | object;
+    // clang-format on
     return j(f, l, x);
   }
 };
diff --git a/schema/sysmon.schema b/schema/sysmon.schema
index 395361179df..7e1a6387279 100644
--- a/schema/sysmon.schema
+++ b/schema/sysmon.schema
@@ -11,7 +11,7 @@ type sysmon.ProcessCreation = record {
     // Process Guid of the process that got spawned/created (child)
     ProcessGuid: string,
     // Process ID used by the os to identify the created process (child)
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process being spawned/created. Considered also the child or source process
     Image: string,
     // Version of the image associated with the main process (child)
@@ -33,9 +33,9 @@ type sysmon.ProcessCreation = record {
     // Logon GUID of the user who created the new process. Value that can help you correlate this event with others that contain the same Logon GUID (Sysmon Events)
     LogonGuid: string,
     // Login ID of the user who created the new process. Value that can help you correlate this event with others that contain the same Logon ID
-    LogonId: string,
+    LogonId: count,
     // ID of the session the user belongs to
-    TerminalSessionId: string,
+    TerminalSessionId: count,
     // Integrity label assigned to a process
     IntegrityLevel: string,
     // Hashes captured by sysmon driver
@@ -43,7 +43,7 @@ type sysmon.ProcessCreation = record {
     // ProcessGUID of the process that spawned/created the main process (child)
     ParentProcessGuid: string,
     // Process ID of the process that spawned/created the main process (child)
-    ParentProcessId: string,
+    ParentProcessId: count,
     // File path that spawned/created the main process
     ParentImage: string,
     // Arguments which were passed to the executable associated with the parent process
@@ -67,15 +67,15 @@ type sysmon.ProcessAccess = record {
     // Process Guid of the source process that opened another process. It is derived from a truncated part of the machine GUID, the process start-time and the process token ID.
     SourceProcessGuid: string,
     // Process ID used by the os to identify the source process that opened another process. Derived partially from the EPROCESS kernel structure
-    SourceProcessId: string,
+    SourceProcessId: count,
     // ID of the specific thread inside of the source process that opened another process
-    SourceThreadId: string,
+    SourceThreadId: count,
     // File path of the source process that created a thread in another process
     SourceImage: string,
     // Process Guid of the target process
     TargetProcessGuid: string,
     // Process ID used by the os to identify the target process
-    TargetProcessId: string,
+    TargetProcessId: count,
     // File path of the target process
     TargetImage: string,
     // The access flags (bitmask) associated with the process rights requested for the target process
@@ -96,7 +96,7 @@ type sysmon.FileCreate = record {
     // Process Guid of the process that created the file
     ProcessGuid: string,
     // Process ID used by the os to identify the process that created the file (child)
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that created the file
     Image: string,
     // Name of the file
@@ -118,7 +118,7 @@ type sysmon.RegistryEventObjectCreateAndDelete = record {
     // Process Guid of the process that created or deleted a registry key
     ProcessGuid: string,
     // Process ID used by the os to identify the process that created or deleted a registry key
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that created or deleted a registry key
     Image: string,
     // complete path of the registry key
@@ -137,7 +137,7 @@ type sysmon.RegistryEventValueSet = record {
     // Process Guid of the process that modified a registry value
     ProcessGuid: string,
     // Process ID used by the os to identify the process that that modified a registry value
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that that modified a registry value
     Image: string,
     // complete path of the registry key
@@ -158,7 +158,7 @@ type sysmon.RegistryEventKeyAndValueRename = record {
     // Process Guid of the process that renamed a registry value and key
     ProcessGuid: string,
     // Process ID used by the os to identify the process that renamed a registry value and key
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that renamed a registry value and key
     Image: string,
     // complete path of the registry key
@@ -181,7 +181,7 @@ type sysmon.FileCreateStreamHash = record {
     // Process Guid of the process that created the named file stream
     ProcessGuid: string,
     // Process ID used by the os to identify the process that created the named file stream
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that created the named file stream
     Image: string,
     // Name of the file
@@ -214,7 +214,7 @@ type sysmon.PipeCreated = record {
     // Process Guid of the process that created the pipe
     ProcessGuid: string,
     // Process ID used by the os to identify the process that created the pipe
-    ProcessId: string,
+    ProcessId: count,
     // Name of the pipe created
     PipeName: string,
     // File path of the process that created the pipe
@@ -233,7 +233,7 @@ type sysmon.PipeConnected = record {
     // Process Guid of the process that connected the pipe
     ProcessGuid: string,
     // Process ID used by the os to identify the process that connected the pipe
-    ProcessId: string,
+    ProcessId: count,
     // Name of the pipe connecged
     PipeName: string,
     // File path of the process that connected the pipe
@@ -276,7 +276,7 @@ type sysmon.ProcessChangedFileCreationTime = record {
     // Process Guid of the process that changed the file creation time
     ProcessGuid: string,
     // Process ID used by the os to identify the process changing the file creation time
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that changed the file creation time
     Image: string,
     // full path name of the file
@@ -357,7 +357,7 @@ type sysmon.FileDelete = record {
     // Process Guid of the process that deleted the file
     ProcessGuid: string,
     // Process ID used by the os to identify the process that deleted the file
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that deleted the file
     Image: string,
     // Name of the account who deleted the file.
@@ -384,7 +384,7 @@ type sysmon.NetworkConnection = record {
     // Process Guid of the process that made the network connection
     ProcessGuid: string,
     // Process ID used by the os to identify the process that made the network connection
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that made the network connection
     Image: string,
     // Name of the account who made the network connection. It usually containes domain name and user name
@@ -400,7 +400,7 @@ type sysmon.NetworkConnection = record {
     // name of the host that made the network connection
     SourceHostname: string,
     // source port number
-    SourcePort: string,
+    SourcePort: count,
     // name of the source port being used (i.e. netbios-dgm)
     SourcePortName: string,
     // is the destination ip an Ipv6
@@ -410,7 +410,7 @@ type sysmon.NetworkConnection = record {
     // name of the host that received the network connection
     DestinationHostname: string,
     // destination port number
-    DestinationPort: string,
+    DestinationPort: count,
     // name of the destination port
     DestinationPortName: string,
 }
@@ -438,7 +438,7 @@ type sysmon.ProcessTerminated = record {
     // Process Guid of the process that terminated
     ProcessGuid: string,
     // Process ID used by the os to identify the process that terminated
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that terminated
     Image: string,
 }
@@ -479,7 +479,7 @@ type sysmon.ImageLoaded = record {
     // Process Guid of the process that loaded the image
     ProcessGuid: string,
     // Process ID used by the os to identify the process that loaded the image
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that loaded the image
     Image: string,
     // full path of the image loaded
@@ -519,17 +519,17 @@ type sysmon.CreateRemoteThread = record {
     // Process Guid of the source process that created a thread in another process
     SourceProcessGuid: string,
     // Process ID used by the os to identify the source process that created a thread in another process
-    SourceProcessId: string,
+    SourceProcessId: count,
     // File path of the source process that created a thread in another process
     SourceImage: string,
     // Process Guid of the target process
     TargetProcessGuid: string,
     // Process ID used by the os to identify the target process
-    TargetProcessId: string,
+    TargetProcessId: count,
     // File path of the target process
     TargetImage: string,
     // Id of the new thread created in the target process
-    NewThreadId: string,
+    NewThreadId: count,
     // New thread start address
     StartAddress: string,
     // Start module determined from thread start address mapping to PEB loaded module list
@@ -551,7 +551,7 @@ type sysmon.RawAccessRead = record {
     // Process Guid of the process that conducted reading operations from the drive
     ProcessGuid: string,
     // Process ID used by the os to identify the process that conducted reading operations from the drive
-    ProcessId: string,
+    ProcessId: count,
     // File path of the process that conducted reading operations from the drive
     Image: string,
     // Target device
diff --git a/scripts/generate-sysmon-schema.py b/scripts/generate-sysmon-schema.py
index 1c842921051..407daef3cde 100755
--- a/scripts/generate-sysmon-schema.py
+++ b/scripts/generate-sysmon-schema.py
@@ -87,7 +87,7 @@ def map_type(name):
     if name == "string":
         return "string"
     elif name == "integer":
-        return "string" # sysmon sends some integers as strings
+        return "count"
     elif name == "date":
         return "time #timestamp"
     elif name == "boolean" or name == "bool":

From 0b313bed0effa6afadad1d4bf5a20b7aa99c0102 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Fri, 29 May 2020 14:51:45 +0200
Subject: [PATCH 04/10] Add changelog entry

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2101679ed27..bf33855b457 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,11 @@ Every entry has a category for which we use the following visual abbreviations:
 
 ## Unreleased
 
+- 🎁 The `import json` command's type restrictions are more relaxed now, and can
+  additionally convert from JSON strings to VAST internal data types. The new
+  option `--strict` allows for restoring the old behavior.
+  [#891](https://github.com/tenzir/vast/pull/891)
+
 - 🎁 VAST now supports /etc/vast/vast.conf as an additional fallback for the
   configuration file. The following file locations are looked at in order: Path
   specified on the command line via `--config=path/to/vast.conf`, `vast.conf` in

From b2aae9547df46a031cf294a2149a9c195882a655 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Fri, 29 May 2020 16:27:18 +0200
Subject: [PATCH 05/10] Use enum class over boolean parameter

---
 libvast/src/format/json.cpp  | 12 +++++++++---
 libvast/test/format/json.cpp |  2 +-
 libvast/vast/format/json.hpp | 13 ++++++++-----
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp
index db42fb2413c..df11a1719c5 100644
--- a/libvast/src/format/json.cpp
+++ b/libvast/src/format/json.cpp
@@ -216,7 +216,7 @@ const char* writer::name() const {
 }
 
 caf::error add(table_slice_builder& builder, const vast::json::object& xs,
-               const record_type& layout, bool strict) {
+               const record_type& layout, conversion_policy policy) {
   for (auto& field : layout.fields) {
     auto i = lookup(field.name, xs);
     // Non-existing fields are treated as empty (unset).
@@ -226,8 +226,14 @@ caf::error add(table_slice_builder& builder, const vast::json::object& xs,
                                            "slice builder");
       continue;
     }
-    auto x = strict ? caf::visit(strict_convert{}, *i, field.type)
-                    : caf::visit(relaxed_convert{}, *i, field.type);
+    auto x = [&] {
+      switch (policy) {
+        case conversion_policy::relaxed:
+          return caf::visit(relaxed_convert{}, *i, field.type);
+        case conversion_policy::strict:
+          return caf::visit(strict_convert{}, *i, field.type);
+      }
+    }();
     if (!x)
       return make_error(ec::convert_error, x.error().context(),
                         "could not convert", field.name, ":", to_string(*i));
diff --git a/libvast/test/format/json.cpp b/libvast/test/format/json.cpp
index b23201c969f..0adf3d223f0 100644
--- a/libvast/test/format/json.cpp
+++ b/libvast/test/format/json.cpp
@@ -104,7 +104,7 @@ TEST(json to data) {
   })json";
   auto jn = unbox(to<json>(str));
   auto xs = caf::get<json::object>(jn);
-  format::json::add(builder, xs, flat, true);
+  format::json::add(builder, xs, flat, format::json::conversion_policy::strict);
   auto ptr = builder.finish();
   REQUIRE(ptr);
   CHECK(ptr->at(0, 11) == data{enumeration{2}});
diff --git a/libvast/vast/format/json.hpp b/libvast/vast/format/json.hpp
index 42d0e37a885..f65c454f91d 100644
--- a/libvast/vast/format/json.hpp
+++ b/libvast/vast/format/json.hpp
@@ -35,6 +35,8 @@
 
 namespace vast::format::json {
 
+enum class conversion_policy { relaxed, strict };
+
 class writer : public ostream_writer {
 public:
   using defaults = vast::defaults::export_::json;
@@ -52,10 +54,10 @@ class writer : public ostream_writer {
 /// @param builder The builder to add the JSON object to.
 /// @param xs The JSON object to add to *builder.
 /// @param layout The record type describing *xs*.
-/// @param strict Toggle between strict and relaxed type matching.
+/// @param policy Type conversion policy.
 /// @returns An error iff the operation failed.
 caf::error add(table_slice_builder& builder, const vast::json::object& xs,
-               const record_type& layout, bool strict);
+               const record_type& layout, conversion_policy policy);
 
 /// @relates reader
 struct default_selector {
@@ -149,7 +151,7 @@ class reader final : public multi_layout_reader {
   mutable size_t num_invalid_lines_ = 0;
   mutable size_t num_unknown_layouts_ = 0;
   mutable size_t num_lines_ = 0;
-  bool strict_ = false;
+  conversion_policy conversion_policy_ = conversion_policy::relaxed;
 };
 
 // -- implementation ----------------------------------------------------------
@@ -162,7 +164,8 @@ reader<Selector>::reader(caf::atom_value table_slice_type,
   if (in != nullptr)
     reset(std::move(in));
   std::string category = vast::defaults::import::json::category;
-  strict_ = caf::get_or(options, category + ".strict", false);
+  if (caf::get_or(options, category + ".strict", false))
+    conversion_policy_ = conversion_policy::strict;
 }
 
 template <class Selector>
@@ -261,7 +264,7 @@ caf::error reader<Selector>::read_impl(size_t max_events, size_t max_slice_size,
     bptr = builder(*layout);
     if (bptr == nullptr)
       return make_error(ec::parse_error, "unable to get a builder");
-    if (auto err = add(*bptr, *xs, *layout, strict_)) {
+    if (auto err = add(*bptr, *xs, *layout, conversion_policy_)) {
       err.context() += caf::make_message("line", lines_->line_number());
       return finish(cons, err);
     }

From af3ae5cabc47ce92a0b97d442455fcea565e24a0 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Wed, 3 Jun 2020 15:50:46 +0200
Subject: [PATCH 06/10] Add a hexadecimal integral parser

---
 libvast/src/format/json.cpp                   | 38 ++++++++--------
 libvast/test/format/json.cpp                  | 15 +++++++
 libvast/test/parseable.cpp                    | 20 ++++++++-
 .../concept/parseable/numeric/integral.hpp    | 43 +++++++++++++------
 libvast/vast/concept/parseable/vast/json.hpp  | 34 ++++++++-------
 5 files changed, 104 insertions(+), 46 deletions(-)

diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp
index df11a1719c5..c9dbb720856 100644
--- a/libvast/src/format/json.cpp
+++ b/libvast/src/format/json.cpp
@@ -148,34 +148,38 @@ struct strict_convert {
 struct relaxed_convert : strict_convert {
   using strict_convert::operator();
 
-  caf::expected<data> operator()(json::string str, const bool_type&) const {
-    if (bool res; parsers::json_boolean(str, res))
-      return res;
+  caf::expected<data>
+  operator()(const json::string& str, const bool_type&) const {
+    if (bool x; parsers::json_boolean(str, x))
+      return x;
     return make_error(ec::convert_error, "cannot convert from", str, "to bool");
   }
 
-  caf::expected<data> operator()(json::string str, const real_type&) const {
-    if (real res; parsers::json_number(str, res))
-      return res;
+  caf::expected<data>
+  operator()(const json::string& str, const real_type&) const {
+    if (real x; parsers::json_number(str, x))
+      return x;
     return make_error(ec::convert_error, "cannot convert from", str, "to real");
   }
 
-  caf::expected<data> operator()(json::string str, const integer_type&) const {
-    if (integer res; parsers::json_int(str, res))
-      return res;
-    if (real res; parsers::json_number(str, res)) {
+  caf::expected<data>
+  operator()(const json::string& str, const integer_type&) const {
+    if (integer x; parsers::json_int(str, x))
+      return x;
+    if (real x; parsers::json_number(str, x)) {
       VAST_WARNING_ANON("json-reader narrowed", str, "to type int");
-      return detail::narrow_cast<integer>(res);
+      return detail::narrow_cast<integer>(x);
     }
     return make_error(ec::convert_error, "cannot convert from", str, "to int");
   }
 
-  caf::expected<data> operator()(json::string str, const count_type&) const {
-    if (count res; parsers::json_count(str, res))
-      return res;
-    if (real res; parsers::json_number(str, res)) {
-      VAST_WARNING_ANON("json-reader narrowed", str, "to type int");
-      return detail::narrow_cast<count>(res);
+  caf::expected<data>
+  operator()(const json::string& str, const count_type&) const {
+    if (count x; parsers::json_count(str, x))
+      return x;
+    if (real x; parsers::json_number(str, x)) {
+      VAST_WARNING_ANON("json-reader narrowed", str, "to type count");
+      return detail::narrow_cast<count>(x);
     }
     return make_error(ec::convert_error, "cannot convert from", str,
                       "to count");
diff --git a/libvast/test/format/json.cpp b/libvast/test/format/json.cpp
index 0adf3d223f0..001c7ae5874 100644
--- a/libvast/test/format/json.cpp
+++ b/libvast/test/format/json.cpp
@@ -131,4 +131,19 @@ TEST_DISABLED(suricata) {
   CHECK(slices[0]->at(0, 19) == data{count{4520}});
 }
 
+TEST(json hex number parser) {
+  using namespace parsers;
+  double x;
+  CHECK(json_number("123.0", x));
+  CHECK_EQUAL(x, 123.0);
+  CHECK(json_number("-123.0", x));
+  CHECK_EQUAL(x, -123.0);
+  CHECK(json_number("123", x));
+  CHECK_EQUAL(x, 123.0);
+  CHECK(json_number("+123", x));
+  CHECK_EQUAL(x, 123.0);
+  CHECK(json_number("0xFF", x));
+  CHECK_EQUAL(x, 255.0);
+}
+
 FIXTURE_SCOPE_END()
diff --git a/libvast/test/parseable.cpp b/libvast/test/parseable.cpp
index 4d19ac14039..3bc68600dc0 100644
--- a/libvast/test/parseable.cpp
+++ b/libvast/test/parseable.cpp
@@ -501,6 +501,24 @@ TEST(unsigned integral) {
   CHECK_EQUAL(x, 12u);
 }
 
+TEST(unsigned hexadecimal integral) {
+  using namespace parsers;
+  auto p = ignore(-hex_prefix) >> hex64;
+  unsigned x;
+  CHECK(p("1234", x));
+  CHECK_EQUAL(x, 0x1234);
+  CHECK(p("13BFC3d1", x));
+  CHECK_EQUAL(x, 0x13BFC3d1);
+  CHECK(p("FF", x));
+  CHECK_EQUAL(x, 0xFF);
+  CHECK(p("ff00", x));
+  CHECK_EQUAL(x, 0xff00);
+  CHECK(p("0X12ab", x));
+  CHECK_EQUAL(x, 0X12ab);
+  CHECK(p("0x3e7", x));
+  CHECK_EQUAL(x, 0x3e7);
+}
+
 TEST(signed integral with digit constraints) {
   constexpr auto max = 4;
   constexpr auto min = 2;
@@ -587,7 +605,7 @@ TEST(byte) {
   auto f = str.begin();
   auto l = f + 1;
   auto u8 = uint8_t{0};
-  CHECK(byte(f, l, u8));
+  CHECK(parsers::byte(f, l, u8));
   CHECK(u8 == 0x01u);
   CHECK(f == l);
   MESSAGE("big endian");
diff --git a/libvast/vast/concept/parseable/numeric/integral.hpp b/libvast/vast/concept/parseable/numeric/integral.hpp
index a8c954e2e94..14f4ae246a7 100644
--- a/libvast/vast/concept/parseable/numeric/integral.hpp
+++ b/libvast/vast/concept/parseable/numeric/integral.hpp
@@ -13,9 +13,10 @@
 
 #pragma once
 
-#include <cstdint>
+#include "vast/concept/parseable/core.hpp"
+#include "vast/detail/coding.hpp"
 
-#include "vast/concept/parseable/core/parser.hpp"
+#include <cstdint>
 
 namespace vast {
 namespace detail {
@@ -32,15 +33,14 @@ bool parse_sign(Iterator& i) {
 
 } // namespace detail
 
-template <
-  class T,
-  int MaxDigits = std::numeric_limits<T>::digits10 + 1,
-  int MinDigits = 1,
-  int Radix = 10
->
+template <class T,
+          // Note that it is fine to have this specific for base 10, as other
+          // bases must be specified as a later template parameters anyways.
+          int MaxDigits = std::numeric_limits<T>::digits10 + 1,
+          int MinDigits = 1, int Radix = 10>
 struct integral_parser
   : parser<integral_parser<T, MaxDigits, MinDigits, Radix>> {
-  static_assert(Radix == 10, "unsupported radix");
+  static_assert(Radix == 10 || Radix == 16, "unsupported radix");
   static_assert(MinDigits > 0, "need at least one minimum digit");
   static_assert(MaxDigits > 0, "need at least one maximum digit");
   static_assert(MinDigits <= MaxDigits, "maximum cannot exceed minimum");
@@ -48,7 +48,13 @@ struct integral_parser
   using attribute = T;
 
   static bool isdigit(char c) {
-    return c >= '0' && c <= '9';
+    if constexpr (Radix == 10)
+      return c >= '0' && c <= '9';
+    else if constexpr (Radix == 16)
+      return std::isxdigit(c);
+    else
+      static_assert(detail::always_false_v<decltype(Radix)>, "unsupported "
+                                                             "radix");
   }
 
   template <class Iterator, class Attribute, class F>
@@ -56,8 +62,15 @@ struct integral_parser
     if (f == l)
       return false;
     int digits = 0;
-    for (a = 0; isdigit(*f) && f != l && digits < MaxDigits; ++f, ++digits)
-      acc(a, *f - '0');
+    for (a = 0; isdigit(*f) && f != l && digits < MaxDigits; ++f, ++digits) {
+      if constexpr (Radix == 10)
+        acc(a, *f - '0');
+      else if constexpr (Radix == 16)
+        acc(a, detail::hex_to_byte(*f));
+      else
+        static_assert(detail::always_false_v<decltype(Radix)>, "unsupported "
+                                                               "radix");
+    }
     return digits >= MinDigits;
   }
 
@@ -141,6 +154,12 @@ auto const u16 = integral_parser<uint16_t>{};
 auto const u32 = integral_parser<uint32_t>{};
 auto const u64 = integral_parser<uint64_t>{};
 
+auto const hex_prefix = ignore(lit{"0x"} | lit{"0X"});
+auto const hex8 = integral_parser<uint8_t, 2, 1, 16>{};
+auto const hex16 = integral_parser<uint16_t, 4, 1, 16>{};
+auto const hex32 = integral_parser<uint32_t, 8, 1, 16>{};
+auto const hex64 = integral_parser<uint64_t, 16, 1, 16>{};
+
 } // namespace parsers
 } // namespace vast
 
diff --git a/libvast/vast/concept/parseable/vast/json.hpp b/libvast/vast/concept/parseable/vast/json.hpp
index 18848293dea..0b1386c3a1a 100644
--- a/libvast/vast/concept/parseable/vast/json.hpp
+++ b/libvast/vast/concept/parseable/vast/json.hpp
@@ -20,20 +20,22 @@
 #include "vast/concept/parseable/numeric/real.hpp"
 #include "vast/concept/parseable/string/char_class.hpp"
 #include "vast/concept/parseable/string/quoted_string.hpp"
+#include "vast/detail/narrow.hpp"
 #include "vast/json.hpp"
 
 namespace vast {
 
 namespace parsers {
 
-static auto const json_boolean = ignore(*parsers::space) >> parsers::boolean;
+static auto const json_boolean = parsers::boolean;
+static auto const json_int = parsers::i64;
+static auto const json_count = (!parsers::hex_prefix >> parsers::u64)
+                               | (parsers::hex_prefix >> parsers::hex64);
 static auto const json_number
-  = ignore(*parsers::space) >> parsers::real_opt_dot;
-
-// These parsers are only needed for relaxed conversion from JSON string to VAST
-// type, they are not part of the actual JSON specification.
-static auto const json_int = ignore(*parsers::space) >> parsers::i64;
-static auto const json_count = ignore(*parsers::space) >> parsers::u64;
+  = (!parsers::hex_prefix >> parsers::real_opt_dot)
+    | (parsers::hex_prefix >> parsers::hex64->*[](uint64_t x) {
+        return detail::narrow_cast<double>(x);
+      });
 
 } // namespace parsers
 
@@ -52,19 +54,19 @@ struct json_parser : parser<json_parser> {
     auto rbrace = ws >> '}' >> ws;
     auto delim = ws >> ',' >> ws;
     // clang-format off
-    auto null = ws >> "null"_p ->* [] { return json::null{}; };
+    auto null = "null"_p ->* [] { return json::null{}; };
     // clang-format on
-    auto string = ws >> parsers::qqstr;
+    auto string = parsers::qqstr;
     auto array = as<json::array>(lbracket >> ~(ref(j) % delim) >> rbracket);
-    auto key_value = ws >> string >> ws >> ':' >> ws >> ref(j);
+    auto key_value = string >> ws >> ':' >> ws >> ref(j);
     auto object = as<json::object>(lbrace >> ~(key_value % delim) >> rbrace);
     // clang-format off
-    j = null
-      | json_boolean
-      | json_number
-      | string
-      | array
-      | object;
+    j = ws >> (null
+            | json_boolean
+            | json_number
+            | string
+            | array
+            | object);
     // clang-format on
     return j(f, l, x);
   }

From 55d853a48065d5abc7f472fb76db62e7c26b4f97 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Thu, 4 Jun 2020 16:34:34 +0200
Subject: [PATCH 07/10] Allow for relaxed parsing of port from strings

---
 libvast/src/format/json.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp
index c9dbb720856..d3d46be61fb 100644
--- a/libvast/src/format/json.cpp
+++ b/libvast/src/format/json.cpp
@@ -184,6 +184,14 @@ struct relaxed_convert : strict_convert {
     return make_error(ec::convert_error, "cannot convert from", str,
                       "to count");
   }
+
+  caf::expected<data> operator()(json::string str, const port_type&) const {
+    if (port x; parsers::port(str, x))
+      return x;
+    if (port::number_type x; parsers::u16(str, x))
+      return port{x};
+    return make_error(ec::convert_error, "cannot convert from", str, "to port");
+  }
 };
 
 const vast::json* lookup(std::string_view field, const vast::json::object& xs) {

From 628df0330e518a3e8d269513cc65dfc4784e017e Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Wed, 3 Jun 2020 16:55:10 +0200
Subject: [PATCH 08/10] Integrate review feedback

---
 libvast/test/parseable.cpp                   |  2 ++
 libvast/vast/concept/parseable/vast/json.hpp | 33 +++++++++++---------
 libvast/vast/format/json.hpp                 |  5 +--
 3 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/libvast/test/parseable.cpp b/libvast/test/parseable.cpp
index 3bc68600dc0..eebd36a7d58 100644
--- a/libvast/test/parseable.cpp
+++ b/libvast/test/parseable.cpp
@@ -517,6 +517,8 @@ TEST(unsigned hexadecimal integral) {
   CHECK_EQUAL(x, 0X12ab);
   CHECK(p("0x3e7", x));
   CHECK_EQUAL(x, 0x3e7);
+  CHECK(p("0x0000aa", x));
+  CHECK_EQUAL(x, 0x0000aa);
 }
 
 TEST(signed integral with digit constraints) {
diff --git a/libvast/vast/concept/parseable/vast/json.hpp b/libvast/vast/concept/parseable/vast/json.hpp
index 0b1386c3a1a..a20ad71dfec 100644
--- a/libvast/vast/concept/parseable/vast/json.hpp
+++ b/libvast/vast/concept/parseable/vast/json.hpp
@@ -27,15 +27,19 @@ namespace vast {
 
 namespace parsers {
 
-static auto const json_boolean = parsers::boolean;
-static auto const json_int = parsers::i64;
-static auto const json_count = (!parsers::hex_prefix >> parsers::u64)
-                               | (parsers::hex_prefix >> parsers::hex64);
+// clang-format off
+static auto const json_boolean
+  = parsers::boolean;
+static auto const json_int
+  = parsers::i64;
+static auto const json_count
+  = (parsers::hex_prefix >> parsers::hex64)
+  | parsers::u64;
 static auto const json_number
-  = (!parsers::hex_prefix >> parsers::real_opt_dot)
-    | (parsers::hex_prefix >> parsers::hex64->*[](uint64_t x) {
-        return detail::narrow_cast<double>(x);
-      });
+  = (parsers::hex_prefix >> parsers::hex64 ->* [](uint64_t x) {
+        return detail::narrow_cast<json::number>(x); })
+  | parsers::real_opt_dot;
+// clang-format on
 
 } // namespace parsers
 
@@ -61,12 +65,13 @@ struct json_parser : parser<json_parser> {
     auto key_value = string >> ws >> ':' >> ws >> ref(j);
     auto object = as<json::object>(lbrace >> ~(key_value % delim) >> rbrace);
     // clang-format off
-    j = ws >> (null
-            | json_boolean
-            | json_number
-            | string
-            | array
-            | object);
+    j = ws >> ( null
+              | json_boolean
+              | json_number
+              | string
+              | array
+              | object
+              );
     // clang-format on
     return j(f, l, x);
   }
diff --git a/libvast/vast/format/json.hpp b/libvast/vast/format/json.hpp
index f65c454f91d..34b152080ee 100644
--- a/libvast/vast/format/json.hpp
+++ b/libvast/vast/format/json.hpp
@@ -161,10 +161,11 @@ reader<Selector>::reader(caf::atom_value table_slice_type,
                          const caf::settings& options,
                          std::unique_ptr<std::istream> in)
   : super(table_slice_type) {
+  using namespace std::string_literals;
   if (in != nullptr)
     reset(std::move(in));
-  std::string category = vast::defaults::import::json::category;
-  if (caf::get_or(options, category + ".strict", false))
+  auto category = vast::defaults::import::json::category;
+  if (caf::get_or(options, category + ".strict"s, false))
     conversion_policy_ = conversion_policy::strict;
 }
 

From 5ce0f1936fdad4579e9faa425e0e293f50d623fc Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Thu, 4 Jun 2020 16:55:59 +0200
Subject: [PATCH 09/10] Support --strict for all JSON sources

---
 libvast/src/system/application.cpp  | 12 ++++++++----
 libvast/vast/system/application.hpp |  3 +++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/libvast/src/system/application.cpp b/libvast/src/system/application.cpp
index e59dce4ccbe..052719c027f 100644
--- a/libvast/src/system/application.cpp
+++ b/libvast/src/system/application.cpp
@@ -177,12 +177,12 @@ auto make_import_command() {
   import_->add_subcommand("csv", "imports CSV logs from STDIN or file",
                           documentation::vast_import_csv,
                           source_opts("?import.csv"));
-  import_->add_subcommand(
-    "json", "imports JSON with schema", documentation::vast_import_json,
-    source_opts("?import.json").add<bool>("strict", "strict type matching"));
+  import_->add_subcommand("json", "imports JSON with schema",
+                          documentation::vast_import_json,
+                          json_source_opts("?import.json"));
   import_->add_subcommand("suricata", "imports suricata eve json",
                           documentation::vast_import_suricata,
-                          source_opts("?import.suricata"));
+                          json_source_opts("?import.suricata"));
   import_->add_subcommand("syslog", "imports syslog messages",
                           documentation::vast_import_syslog,
                           source_opts("?import.syslog"));
@@ -470,6 +470,10 @@ command::opts_builder source_opts(std::string_view category) {
     .add<bool>("uds,d", "treat -r as listening UNIX domain socket");
 }
 
+command::opts_builder json_source_opts(std::string_view category) {
+  return source_opts(category).add<bool>("strict", "strict type matching");
+}
+
 command::opts_builder sink_opts(std::string_view category) {
   return command::opts(category)
     .add<std::string>("write,w", "path to write events to")
diff --git a/libvast/vast/system/application.hpp b/libvast/vast/system/application.hpp
index 6b92c5b7b76..86edf141fcc 100644
--- a/libvast/vast/system/application.hpp
+++ b/libvast/vast/system/application.hpp
@@ -36,6 +36,9 @@ void render_error(const command& root, const caf::error& err, std::ostream& os);
 /// @returns default options for source commands.
 command::opts_builder source_opts(std::string_view category);
 
+/// @returns default options for JSON source commands.
+command::opts_builder json_source_opts(std::string_view category);
+
 /// @returns defaults options for sink commands.
 command::opts_builder sink_opts(std::string_view category);
 

From a093cce606a9cff94f57bf47c4e5e390f671e790 Mon Sep 17 00:00:00 2001
From: Dominik Lohmann <mail@dominiklohmann.de>
Date: Fri, 5 Jun 2020 10:19:29 +0200
Subject: [PATCH 10/10] Remove --strict option for json import

---
 CHANGELOG.md                        |  3 +--
 libvast/src/format/json.cpp         | 33 ++++++++++-------------------
 libvast/src/system/application.cpp  |  8 ++-----
 libvast/test/format/json.cpp        |  2 +-
 libvast/vast/format/json.hpp        | 15 +++----------
 libvast/vast/system/application.hpp |  3 ---
 vast.conf                           |  3 ---
 7 files changed, 18 insertions(+), 49 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bf33855b457..f57dfc6abdf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,8 +12,7 @@ Every entry has a category for which we use the following visual abbreviations:
 ## Unreleased
 
 - 🎁 The `import json` command's type restrictions are more relaxed now, and can
-  additionally convert from JSON strings to VAST internal data types. The new
-  option `--strict` allows for restoring the old behavior.
+  additionally convert from JSON strings to VAST internal data types.
   [#891](https://github.com/tenzir/vast/pull/891)
 
 - 🎁 VAST now supports /etc/vast/vast.conf as an additional fallback for the
diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp
index d3d46be61fb..3c07ef450d5 100644
--- a/libvast/src/format/json.cpp
+++ b/libvast/src/format/json.cpp
@@ -37,7 +37,7 @@
 namespace vast::format::json {
 namespace {
 
-struct strict_convert {
+struct convert {
   template <class T>
   using expected = caf::expected<T>;
   using json = vast::json;
@@ -136,18 +136,6 @@ struct strict_convert {
     return xs;
   }
 
-  template <class T, class U>
-  caf::expected<data> operator()(T, U) const {
-    VAST_ERROR_ANON("json-reader cannot convert from",
-                    caf::detail::pretty_type_name(typeid(T)), "to",
-                    caf::detail::pretty_type_name(typeid(U)));
-    return make_error(ec::syntax_error, "invalid json type");
-  }
-};
-
-struct relaxed_convert : strict_convert {
-  using strict_convert::operator();
-
   caf::expected<data>
   operator()(const json::string& str, const bool_type&) const {
     if (bool x; parsers::json_boolean(str, x))
@@ -192,6 +180,14 @@ struct relaxed_convert : strict_convert {
       return port{x};
     return make_error(ec::convert_error, "cannot convert from", str, "to port");
   }
+
+  template <class T, class U>
+  caf::expected<data> operator()(T, U) const {
+    VAST_ERROR_ANON("json-reader cannot convert from",
+                    caf::detail::pretty_type_name(typeid(T)), "to",
+                    caf::detail::pretty_type_name(typeid(U)));
+    return make_error(ec::syntax_error, "invalid json type");
+  }
 };
 
 const vast::json* lookup(std::string_view field, const vast::json::object& xs) {
@@ -228,7 +224,7 @@ const char* writer::name() const {
 }
 
 caf::error add(table_slice_builder& builder, const vast::json::object& xs,
-               const record_type& layout, conversion_policy policy) {
+               const record_type& layout) {
   for (auto& field : layout.fields) {
     auto i = lookup(field.name, xs);
     // Non-existing fields are treated as empty (unset).
@@ -238,14 +234,7 @@ caf::error add(table_slice_builder& builder, const vast::json::object& xs,
                                            "slice builder");
       continue;
     }
-    auto x = [&] {
-      switch (policy) {
-        case conversion_policy::relaxed:
-          return caf::visit(relaxed_convert{}, *i, field.type);
-        case conversion_policy::strict:
-          return caf::visit(strict_convert{}, *i, field.type);
-      }
-    }();
+    auto x = caf::visit(convert{}, *i, field.type);
     if (!x)
       return make_error(ec::convert_error, x.error().context(),
                         "could not convert", field.name, ":", to_string(*i));
diff --git a/libvast/src/system/application.cpp b/libvast/src/system/application.cpp
index 052719c027f..ec531e68a15 100644
--- a/libvast/src/system/application.cpp
+++ b/libvast/src/system/application.cpp
@@ -179,10 +179,10 @@ auto make_import_command() {
                           source_opts("?import.csv"));
   import_->add_subcommand("json", "imports JSON with schema",
                           documentation::vast_import_json,
-                          json_source_opts("?import.json"));
+                          source_opts("?import.json"));
   import_->add_subcommand("suricata", "imports suricata eve json",
                           documentation::vast_import_suricata,
-                          json_source_opts("?import.suricata"));
+                          source_opts("?import.suricata"));
   import_->add_subcommand("syslog", "imports syslog messages",
                           documentation::vast_import_syslog,
                           source_opts("?import.syslog"));
@@ -470,10 +470,6 @@ command::opts_builder source_opts(std::string_view category) {
     .add<bool>("uds,d", "treat -r as listening UNIX domain socket");
 }
 
-command::opts_builder json_source_opts(std::string_view category) {
-  return source_opts(category).add<bool>("strict", "strict type matching");
-}
-
 command::opts_builder sink_opts(std::string_view category) {
   return command::opts(category)
     .add<std::string>("write,w", "path to write events to")
diff --git a/libvast/test/format/json.cpp b/libvast/test/format/json.cpp
index 001c7ae5874..a4c61e4db08 100644
--- a/libvast/test/format/json.cpp
+++ b/libvast/test/format/json.cpp
@@ -104,7 +104,7 @@ TEST(json to data) {
   })json";
   auto jn = unbox(to<json>(str));
   auto xs = caf::get<json::object>(jn);
-  format::json::add(builder, xs, flat, format::json::conversion_policy::strict);
+  format::json::add(builder, xs, flat);
   auto ptr = builder.finish();
   REQUIRE(ptr);
   CHECK(ptr->at(0, 11) == data{enumeration{2}});
diff --git a/libvast/vast/format/json.hpp b/libvast/vast/format/json.hpp
index 34b152080ee..c439e3c04b6 100644
--- a/libvast/vast/format/json.hpp
+++ b/libvast/vast/format/json.hpp
@@ -31,12 +31,9 @@
 
 #include <caf/expected.hpp>
 #include <caf/fwd.hpp>
-#include <caf/settings.hpp>
 
 namespace vast::format::json {
 
-enum class conversion_policy { relaxed, strict };
-
 class writer : public ostream_writer {
 public:
   using defaults = vast::defaults::export_::json;
@@ -54,10 +51,9 @@ class writer : public ostream_writer {
 /// @param builder The builder to add the JSON object to.
 /// @param xs The JSON object to add to *builder.
 /// @param layout The record type describing *xs*.
-/// @param policy Type conversion policy.
 /// @returns An error iff the operation failed.
 caf::error add(table_slice_builder& builder, const vast::json::object& xs,
-               const record_type& layout, conversion_policy policy);
+               const record_type& layout);
 
 /// @relates reader
 struct default_selector {
@@ -123,7 +119,7 @@ class reader final : public multi_layout_reader {
   /// @param options Additional options.
   /// @param in The stream of JSON objects.
   explicit reader(caf::atom_value table_slice_type,
-                  const caf::settings& options,
+                  const caf::settings& /*options*/,
                   std::unique_ptr<std::istream> in = nullptr);
 
   void reset(std::unique_ptr<std::istream> in);
@@ -151,7 +147,6 @@ class reader final : public multi_layout_reader {
   mutable size_t num_invalid_lines_ = 0;
   mutable size_t num_unknown_layouts_ = 0;
   mutable size_t num_lines_ = 0;
-  conversion_policy conversion_policy_ = conversion_policy::relaxed;
 };
 
 // -- implementation ----------------------------------------------------------
@@ -161,12 +156,8 @@ reader<Selector>::reader(caf::atom_value table_slice_type,
                          const caf::settings& options,
                          std::unique_ptr<std::istream> in)
   : super(table_slice_type) {
-  using namespace std::string_literals;
   if (in != nullptr)
     reset(std::move(in));
-  auto category = vast::defaults::import::json::category;
-  if (caf::get_or(options, category + ".strict"s, false))
-    conversion_policy_ = conversion_policy::strict;
 }
 
 template <class Selector>
@@ -265,7 +256,7 @@ caf::error reader<Selector>::read_impl(size_t max_events, size_t max_slice_size,
     bptr = builder(*layout);
     if (bptr == nullptr)
       return make_error(ec::parse_error, "unable to get a builder");
-    if (auto err = add(*bptr, *xs, *layout, conversion_policy_)) {
+    if (auto err = add(*bptr, *xs, *layout)) {
       err.context() += caf::make_message("line", lines_->line_number());
       return finish(cons, err);
     }
diff --git a/libvast/vast/system/application.hpp b/libvast/vast/system/application.hpp
index 86edf141fcc..6b92c5b7b76 100644
--- a/libvast/vast/system/application.hpp
+++ b/libvast/vast/system/application.hpp
@@ -36,9 +36,6 @@ void render_error(const command& root, const caf::error& err, std::ostream& os);
 /// @returns default options for source commands.
 command::opts_builder source_opts(std::string_view category);
 
-/// @returns default options for JSON source commands.
-command::opts_builder json_source_opts(std::string_view category);
-
 /// @returns defaults options for sink commands.
 command::opts_builder sink_opts(std::string_view category);
 
diff --git a/vast.conf b/vast.conf
index 09b01b841a9..507d0a11390 100644
--- a/vast.conf
+++ b/vast.conf
@@ -77,9 +77,6 @@ export {
   ; The `vast export json` command exports events formatted as JSONL (line-
   ; delimited JSON).
   json {
-    ; Strict type matching.
-    ;strict = false
-
     ; For additionally available options, see export.ascii.
   }