Add support for compressing CLP IR streams.

.gitmodules

@@ -11,3 +11,6 @@
 [submodule "components/core/submodules/yaml-cpp"]
 	path = components/core/submodules/yaml-cpp
 	url = https://github.com/jbeder/yaml-cpp.git
+[submodule "components/core/submodules/boost-outcome"]
+	path = components/core/submodules/boost-outcome
+	url = git@github.com:boostorg/outcome.git

components/core/CMakeLists.txt

@@ -227,6 +227,15 @@ set(SOURCE_FILES_clp
         src/EncodedVariableInterpreter.cpp
         src/EncodedVariableInterpreter.hpp
         src/ErrorCode.hpp
+        src/ffi/encoding_methods.cpp
+        src/ffi/encoding_methods.hpp
+        src/ffi/encoding_methods.tpp
+        src/ffi/ir_stream/byteswap.hpp
+        src/ffi/ir_stream/decoding_methods.cpp
+        src/ffi/ir_stream/decoding_methods.hpp
+        src/ffi/ir_stream/decoding_methods.tpp
+        src/ffi/ir_stream/encoding_methods.cpp
+        src/ffi/ir_stream/encoding_methods.hpp
         src/FileReader.cpp
         src/FileReader.hpp
         src/FileWriter.cpp
@@ -239,6 +248,11 @@ set(SOURCE_FILES_clp
         src/GlobalMySQLMetadataDB.hpp
         src/GlobalSQLiteMetadataDB.cpp
         src/GlobalSQLiteMetadataDB.hpp
+        src/ir/LogEvent.hpp
+        src/ir/LogEventDeserializer.cpp
+        src/ir/LogEventDeserializer.hpp
+        src/ir/utils.cpp
+        src/ir/utils.hpp
         src/LibarchiveFileReader.cpp
         src/LibarchiveFileReader.hpp
         src/LibarchiveReader.cpp
@@ -397,6 +411,12 @@ set(SOURCE_FILES_clg
         src/EncodedVariableInterpreter.cpp
         src/EncodedVariableInterpreter.hpp
         src/ErrorCode.hpp
+        src/ffi/encoding_methods.cpp
+        src/ffi/encoding_methods.hpp
+        src/ffi/encoding_methods.tpp
+        src/ffi/ir_stream/decoding_methods.cpp
+        src/ffi/ir_stream/decoding_methods.hpp
+        src/ffi/ir_stream/decoding_methods.tpp
         src/FileReader.cpp
         src/FileReader.hpp
         src/FileWriter.cpp
@@ -411,6 +431,7 @@ set(SOURCE_FILES_clg
         src/GlobalSQLiteMetadataDB.hpp
         src/Grep.cpp
         src/Grep.hpp
+        src/ir/LogEvent.hpp
         src/LogTypeDictionaryEntry.cpp
         src/LogTypeDictionaryEntry.hpp
         src/LogTypeDictionaryReader.cpp
@@ -554,12 +575,19 @@ set(SOURCE_FILES_clo
         src/EncodedVariableInterpreter.cpp
         src/EncodedVariableInterpreter.hpp
         src/ErrorCode.hpp
+        src/ffi/encoding_methods.cpp
+        src/ffi/encoding_methods.hpp
+        src/ffi/encoding_methods.tpp
+        src/ffi/ir_stream/decoding_methods.cpp
+        src/ffi/ir_stream/decoding_methods.hpp
+        src/ffi/ir_stream/decoding_methods.tpp
         src/FileReader.cpp
         src/FileReader.hpp
         src/FileWriter.cpp
         src/FileWriter.hpp
         src/Grep.cpp
         src/Grep.hpp
+        src/ir/LogEvent.hpp
         src/LogTypeDictionaryEntry.cpp
         src/LogTypeDictionaryEntry.hpp
         src/LogTypeDictionaryReader.cpp
@@ -726,6 +754,7 @@ set(SOURCE_FILES_unitTest
         src/ffi/ir_stream/byteswap.hpp
         src/ffi/ir_stream/decoding_methods.cpp
         src/ffi/ir_stream/decoding_methods.hpp
+        src/ffi/ir_stream/decoding_methods.tpp
         src/ffi/ir_stream/encoding_methods.cpp
         src/ffi/ir_stream/encoding_methods.hpp
         src/ffi/ir_stream/protocol_constants.hpp
@@ -757,6 +786,11 @@ set(SOURCE_FILES_unitTest
         src/GlobalSQLiteMetadataDB.hpp
         src/Grep.cpp
         src/Grep.hpp
+        src/ir/LogEvent.hpp
+        src/ir/LogEventDeserializer.cpp
+        src/ir/LogEventDeserializer.hpp
+        src/ir/utils.cpp
+        src/ir/utils.hpp
         src/LibarchiveFileReader.cpp
         src/LibarchiveFileReader.hpp
         src/LibarchiveReader.cpp

components/core/src/EncodedVariableInterpreter.cpp

@@ -7,6 +7,7 @@
 // Project headers
 #include "Defs.h"
 #include "ffi/encoding_methods.hpp"
+#include "ffi/ir_stream/decoding_methods.hpp"
 #include "spdlog_with_specializations.hpp"
 #include "string_utils.hpp"
 #include "type_utils.hpp"
@@ -207,24 +208,79 @@ void EncodedVariableInterpreter::encode_and_add_to_dictionary (const string& mes
     // To avoid reallocating the logtype as we append to it, reserve enough space to hold the entire message
     logtype_dict_entry.reserve_constant_length(message.length());
     while (logtype_dict_entry.parse_next_var(message, var_begin_pos, var_end_pos, var_str)) {
-        // Encode variable
-        encoded_variable_t encoded_var;
-        if (convert_string_to_representable_integer_var(var_str, encoded_var)) {
-            logtype_dict_entry.add_int_var();
-        } else if (convert_string_to_representable_float_var(var_str, encoded_var)) {
-            logtype_dict_entry.add_float_var();
-        } else {
-            // Variable string looks like a dictionary variable, so encode it as so
-            variable_dictionary_id_t id;
-            var_dict.add_entry(var_str, id);
-            encoded_var = encode_var_dict_id(id);
-            var_ids.push_back(id);
+        auto encoded_var = encode_var(var_str, logtype_dict_entry, var_dict, var_ids);
+        encoded_vars.push_back(encoded_var);
+    }
+}
 
-            logtype_dict_entry.add_dictionary_var();
-        }
+template<typename encoded_variable_t>
+void EncodedVariableInterpreter::encode_and_add_to_dictionary(
+        ir::LogEvent<encoded_variable_t> const& log_event,
+        LogTypeDictionaryEntry& logtype_dict_entry,
+        VariableDictionaryWriter& var_dict,
+        std::vector<ffi::eight_byte_encoded_variable_t>& encoded_vars,
+        std::vector<variable_dictionary_id_t>& var_ids,
+        size_t& raw_num_bytes
+) {
+    logtype_dict_entry.clear();
+    logtype_dict_entry.reserve_constant_length(log_event.get_logtype().length());
 
+    raw_num_bytes = 0;
+
+    auto constant_handler = [&](std::string const& value, size_t begin_pos, size_t length) {
+        raw_num_bytes += length;
+        logtype_dict_entry.add_constant(value, begin_pos, length);
+    };
+
+    auto encoded_int_handler = [&](encoded_variable_t encoded_var) {
+        raw_num_bytes += ffi::decode_integer_var(encoded_var).length();
+        logtype_dict_entry.add_int_var();
+
+        ffi::eight_byte_encoded_variable_t eight_byte_encoded_var{};
+        if constexpr (std::is_same_v<encoded_variable_t, ffi::eight_byte_encoded_variable_t>) {
+            eight_byte_encoded_var = encoded_var;
+        } else {  // std::is_same_v<encoded_variable_t, ffi::four_byte_encoded_variable_t>
+            eight_byte_encoded_var = ffi::encode_four_byte_integer_as_eight_byte(encoded_var);
+        }
+        encoded_vars.push_back(eight_byte_encoded_var);
+    };
+
+    auto encoded_float_handler = [&](ffi::four_byte_encoded_variable_t encoded_var) {
+        raw_num_bytes += ffi::decode_float_var(encoded_var).length();
+        logtype_dict_entry.add_float_var();
+
+        ffi::eight_byte_encoded_variable_t eight_byte_encoded_var{};
+        if constexpr (std::is_same_v<encoded_variable_t, ffi::eight_byte_encoded_variable_t>) {
+            eight_byte_encoded_var = encoded_var;
+        } else {  // std::is_same_v<encoded_variable_t, ffi::four_byte_encoded_variable_t>
+            eight_byte_encoded_var = ffi::encode_four_byte_float_as_eight_byte(encoded_var);
+        }
+        encoded_vars.push_back(eight_byte_encoded_var);
+    };
+
+    auto dict_var_handler = [&](string const& dict_var) {
+        raw_num_bytes += dict_var.length();
+
+        ffi::eight_byte_encoded_variable_t encoded_var{};
+        if constexpr (std::is_same_v<encoded_variable_t, ffi::eight_byte_encoded_variable_t>) {
+            encoded_var = encode_var_dict_id(
+                    add_dict_var(dict_var, logtype_dict_entry, var_dict, var_ids)
+            );
+        } else {  // std::is_same_v<encoded_variable_t, ffi::four_byte_encoded_variable_t>
+            encoded_var = encode_var(dict_var, logtype_dict_entry, var_dict, var_ids);
+        }
         encoded_vars.push_back(encoded_var);
-    }
+    };
+
+    ffi::ir_stream::generic_decode_message(
+            log_event.get_logtype(),
+            log_event.get_encoded_vars(),
+            log_event.get_dict_vars(),
+            constant_handler,
+            encoded_int_handler,
+            encoded_float_handler,
+            dict_var_handler
+    );
 }
 
 bool EncodedVariableInterpreter::decode_variables_into_message (const LogTypeDictionaryEntry& logtype_dict_entry, const VariableDictionaryReader& var_dict,
@@ -337,3 +393,58 @@ bool EncodedVariableInterpreter::wildcard_search_dictionary_and_get_encoded_matc
 encoded_variable_t EncodedVariableInterpreter::encode_var_dict_id (variable_dictionary_id_t id) {
     return bit_cast<encoded_variable_t>(id);
 }
+
+encoded_variable_t EncodedVariableInterpreter::encode_var(
+        string const& var,
+        LogTypeDictionaryEntry& logtype_dict_entry,
+        VariableDictionaryWriter& var_dict,
+        vector<variable_dictionary_id_t>& var_ids
+) {
+    encoded_variable_t encoded_var{0};
+    if (convert_string_to_representable_integer_var(var, encoded_var)) {
+        logtype_dict_entry.add_int_var();
+    } else if (convert_string_to_representable_float_var(var, encoded_var)) {
+        logtype_dict_entry.add_float_var();
+    } else {
+        // Variable string looks like a dictionary variable, so encode it as so
+        encoded_var = encode_var_dict_id(add_dict_var(var, logtype_dict_entry, var_dict, var_ids));
+    }
+    return encoded_var;
+}
+
+variable_dictionary_id_t EncodedVariableInterpreter::add_dict_var(
+        string const& var,
+        LogTypeDictionaryEntry& logtype_dict_entry,
+        VariableDictionaryWriter& var_dict,
+        vector<variable_dictionary_id_t>& var_ids
+) {
+    variable_dictionary_id_t id{cVariableDictionaryIdMax};
+    var_dict.add_entry(var, id);
+    var_ids.push_back(id);
+
+    logtype_dict_entry.add_dictionary_var();
+
+    return id;
+}
+
+// Explicitly declare template specializations so that we can define the
+// template methods in this file
+template
+void EncodedVariableInterpreter::encode_and_add_to_dictionary<ffi::eight_byte_encoded_variable_t>(
+        ir::LogEvent<ffi::eight_byte_encoded_variable_t> const& log_event,
+        LogTypeDictionaryEntry& logtype_dict_entry,
+        VariableDictionaryWriter& var_dict,
+        std::vector<ffi::eight_byte_encoded_variable_t>& encoded_vars,
+        std::vector<variable_dictionary_id_t>& var_ids,
+        size_t& raw_num_bytes
+);
+
+template
+void EncodedVariableInterpreter::encode_and_add_to_dictionary<ffi::four_byte_encoded_variable_t >(
+        ir::LogEvent<ffi::four_byte_encoded_variable_t> const& log_event,
+        LogTypeDictionaryEntry& logtype_dict_entry,
+        VariableDictionaryWriter& var_dict,
+        std::vector<ffi::eight_byte_encoded_variable_t>& encoded_vars,
+        std::vector<variable_dictionary_id_t>& var_ids,
+        size_t& raw_num_bytes
+);

components/core/src/EncodedVariableInterpreter.hpp

@@ -6,6 +6,7 @@
 #include <vector>
 
 // Project headers
+#include "ir/LogEvent.hpp"
 #include "Query.hpp"
 #include "TraceableException.hpp"
 #include "VariableDictionaryReader.hpp"
@@ -73,6 +74,32 @@ class EncodedVariableInterpreter {
      */
     static void encode_and_add_to_dictionary (const std::string& message, LogTypeDictionaryEntry& logtype_dict_entry, VariableDictionaryWriter& var_dict,
                                               std::vector<encoded_variable_t>& encoded_vars, std::vector<variable_dictionary_id_t>& var_ids);
+
+    /**
+     * Encodes the given IR log event, constructing a logtype dictionary entry,
+     * and adding any dictionary variables to the dictionary. NOTE: Four-byte
+     * encoded variables will be converted to eight-byte encoded variables.
+     * @tparam encoded_variable_t The type of the encoded variables in the log
+     * event
+     * @param log_event
+     * @param logtype_dict_entry
+     * @param var_dict
+     * @param encoded_vars A container to store the encoded variables in
+     * @param var_ids A container to store the dictionary IDs for dictionary
+     * variables
+     * @param raw_num_bytes Returns an estimate of the number of bytes that
+     * this log event would occupy if it was not encoded in CLP's IR
+     */
+    template<typename encoded_variable_t>
+    static void encode_and_add_to_dictionary(
+            ir::LogEvent<encoded_variable_t> const& log_event,
+            LogTypeDictionaryEntry& logtype_dict_entry,
+            VariableDictionaryWriter& var_dict,
+            std::vector<ffi::eight_byte_encoded_variable_t>& encoded_vars,
+            std::vector<variable_dictionary_id_t>& var_ids,
+            size_t& raw_num_bytes
+    );
+
     /**
      * Decodes all variables and decompresses them into a message
      * @param logtype_dict_entry
@@ -106,6 +133,40 @@ class EncodedVariableInterpreter {
      */
     static bool wildcard_search_dictionary_and_get_encoded_matches (const std::string& var_wildcard_str, const VariableDictionaryReader& var_dict,
                                                                     bool ignore_case, SubQuery& sub_query);
+
+private:
+    /**
+     * Encodes the given string as a dictionary or non-dictionary variable and
+     * adds a corresponding placeholder to the logtype
+     * @param var
+     * @param logtype_dict_entry
+     * @param var_dict
+     * @param var_ids A container to add the dictionary ID to (if the string is
+     * a dictionary variable)
+     * @return The encoded variable
+     */
+    static encoded_variable_t encode_var(
+            std::string const& var,
+            LogTypeDictionaryEntry& logtype_dict_entry,
+            VariableDictionaryWriter& var_dict,
+            std::vector<variable_dictionary_id_t>& var_ids
+    );
+
+    /**
+     * Adds the given string to the variable dictionary and adds a corresponding
+     * placeholder to logtype
+     * @param var
+     * @param logtype_dict_entry
+     * @param var_dict
+     * @param var_ids A container to add the dictionary ID to
+     * @return The dictionary ID
+     */
+    static variable_dictionary_id_t add_dict_var(
+            std::string const& var,
+            LogTypeDictionaryEntry& logtype_dict_entry,
+            VariableDictionaryWriter& var_dict,
+            std::vector<variable_dictionary_id_t>& var_ids
+    );
 };
 
 #endif // ENCODEDVARIABLEINTERPRETER_HPP