Add Support for Logging to JSON (beta feature) (#1112)

* Add support for logging protobuf to JSON.

Co-authored-by: Russell Hancox <russellhancox@users.noreply.github.com>

Source/common/SNTCommonEnums.h

@@ -121,6 +121,7 @@ typedef NS_ENUM(NSInteger, SNTEventLogType) {
   SNTEventLogTypeSyslog,
   SNTEventLogTypeFilelog,
   SNTEventLogTypeProtobuf,
+  SNTEventLogTypeJSON,
   SNTEventLogTypeNull,
 };
 

Source/common/SNTConfigurator.m

@@ -794,6 +794,8 @@ - (SNTEventLogType)eventLogType {
     return SNTEventLogTypeSyslog;
   } else if ([logType isEqualToString:@"null"]) {
     return SNTEventLogTypeNull;
+  } else if ([logType isEqualToString:@"json"]) {
+    return SNTEventLogTypeJSON;
   } else if ([logType isEqualToString:@"file"]) {
     return SNTEventLogTypeFilelog;
   } else {

Source/santad/Logs/EndpointSecurity/Logger.mm

@@ -72,6 +72,11 @@
         Protobuf::Create(esapi, std::move(decision_cache)),
         Spool::Create([spool_log_path UTF8String], spool_dir_size_threshold,
                       spool_file_size_threshold, spool_flush_timeout_ms));
+    case SNTEventLogTypeJSON:
+      return std::make_unique<Logger>(
+        Protobuf::Create(esapi, std::move(decision_cache), true),
+        File::Create(event_log_path, kFlushBufferTimeoutMS, kBufferBatchSizeBytes,
+                     kMaxExpectedWriteSizeBytes));
     default: LOGE(@"Invalid log type: %ld", log_type); return nullptr;
   }
 }

Source/santad/Logs/EndpointSecurity/LoggerTest.mm

@@ -125,6 +125,11 @@ LoggerPeer logger(
                                      @"/tmp/spool", 1, 1, 1));
   XCTAssertNotEqual(nullptr, std::dynamic_pointer_cast<Protobuf>(logger.Serializer()));
   XCTAssertNotEqual(nullptr, std::dynamic_pointer_cast<Spool>(logger.Writer()));
+
+  logger = LoggerPeer(
+    Logger::Create(mockESApi, SNTEventLogTypeJSON, nil, @"/tmp/temppy", @"/tmp/spool", 1, 1, 1));
+  XCTAssertNotEqual(nullptr, std::dynamic_pointer_cast<Protobuf>(logger.Serializer()));
+  XCTAssertNotEqual(nullptr, std::dynamic_pointer_cast<File>(logger.Writer()));
 }
 
 - (void)testLog {

Source/santad/Logs/EndpointSecurity/Serializers/Protobuf.h

@@ -33,11 +33,11 @@ class Protobuf : public Serializer {
  public:
   static std::shared_ptr<Protobuf> Create(
     std::shared_ptr<santa::santad::event_providers::endpoint_security::EndpointSecurityAPI> esapi,
-    SNTDecisionCache *decision_cache);
+    SNTDecisionCache *decision_cache, bool json = false);
 
   Protobuf(
     std::shared_ptr<santa::santad::event_providers::endpoint_security::EndpointSecurityAPI> esapi,
-    SNTDecisionCache *decision_cache);
+    SNTDecisionCache *decision_cache, bool json = false);
 
   std::vector<uint8_t> SerializeMessage(
     const santa::santad::event_providers::endpoint_security::EnrichedClose &) override;
@@ -87,6 +87,9 @@ class Protobuf : public Serializer {
   std::vector<uint8_t> FinalizeProto(::santa::pb::v1::SantaMessage *santa_msg);
 
   std::shared_ptr<santa::santad::event_providers::endpoint_security::EndpointSecurityAPI> esapi_;
+  // Toggle for transforming protobuf output to its JSON form.
+  // See https://protobuf.dev/programming-guides/proto3/#json
+  bool json_;
 };
 
 }  // namespace santa::santad::logs::endpoint_security::serializers

Source/santad/Logs/EndpointSecurity/Serializers/Protobuf.mm

@@ -35,9 +35,12 @@
 #include "Source/santad/Logs/EndpointSecurity/Serializers/Utilities.h"
 #import "Source/santad/SNTDecisionCache.h"
 #include "google/protobuf/timestamp.pb.h"
+#include "google/protobuf/util/json_util.h"
 
 using google::protobuf::Arena;
 using google::protobuf::Timestamp;
+using google::protobuf::util::JsonPrintOptions;
+using google::protobuf::util::MessageToJsonString;
 
 using santa::common::NSStringToUTF8StringView;
 using santa::santad::event_providers::endpoint_security::EndpointSecurityAPI;
@@ -66,12 +69,13 @@
 namespace santa::santad::logs::endpoint_security::serializers {
 
 std::shared_ptr<Protobuf> Protobuf::Create(std::shared_ptr<EndpointSecurityAPI> esapi,
-                                           SNTDecisionCache *decision_cache) {
-  return std::make_shared<Protobuf>(esapi, std::move(decision_cache));
+                                           SNTDecisionCache *decision_cache, bool json) {
+  return std::make_shared<Protobuf>(esapi, std::move(decision_cache), json);
 }
 
-Protobuf::Protobuf(std::shared_ptr<EndpointSecurityAPI> esapi, SNTDecisionCache *decision_cache)
-    : Serializer(std::move(decision_cache)), esapi_(esapi) {}
+Protobuf::Protobuf(std::shared_ptr<EndpointSecurityAPI> esapi, SNTDecisionCache *decision_cache,
+                   bool json)
+    : Serializer(std::move(decision_cache)), esapi_(esapi), json_(json) {}
 
 static inline void EncodeTimestamp(Timestamp *timestamp, struct timespec ts) {
   timestamp->set_seconds(ts.tv_sec);
@@ -387,6 +391,26 @@ static inline void EncodeCertificateInfo(::pbv1::CertificateInfo *pb_cert_info,
 }
 
 std::vector<uint8_t> Protobuf::FinalizeProto(::pbv1::SantaMessage *santa_msg) {
+  if (this->json_) {
+    // TODO: Profile this. It's probably not the most efficient way to do this.
+    JsonPrintOptions options;
+    options.always_print_enums_as_ints = false;
+    options.always_print_primitive_fields = true;
+    options.preserve_proto_field_names = true;
+    std::string json;
+
+    google::protobuf::util::Status status = MessageToJsonString(*santa_msg, &json, options);
+
+    if (!status.ok()) {
+      LOGE(@"Failed to convert protobuf to JSON: %s", status.ToString().c_str());
+    }
+
+    std::vector<uint8_t> vec(json.begin(), json.end());
+    // Add a newline to the end of the JSON row.
+    vec.push_back('\n');
+    return vec;
+  }
+
   std::vector<uint8_t> vec(santa_msg->ByteSizeLong());
   santa_msg->SerializeWithCachedSizesToArray(vec.data());
   return vec;

Source/santad/Logs/EndpointSecurity/Serializers/ProtobufTest.mm

@@ -46,6 +46,7 @@
 
 using google::protobuf::Timestamp;
 using google::protobuf::util::JsonPrintOptions;
+using google::protobuf::util::JsonStringToMessage;
 using santa::santad::event_providers::endpoint_security::EnrichedEventType;
 using santa::santad::event_providers::endpoint_security::EnrichedMessage;
 using santa::santad::event_providers::endpoint_security::Enricher;
@@ -161,10 +162,34 @@ bool CompareTime(const Timestamp &timestamp, struct timespec ts) {
   return json;
 }
 
+NSDictionary *findDelta(NSDictionary *a, NSDictionary *b) {
+  NSMutableDictionary *delta = NSMutableDictionary.dictionary;
+
+  // Find objects in a that don't exist or are different in b.
+  [a enumerateKeysAndObjectsUsingBlock:^(id _Nonnull key, id _Nonnull obj, BOOL *_Nonnull stop) {
+    id otherObj = b[key];
+
+    if (![obj isEqual:otherObj]) {
+      delta[key] = obj;
+    }
+  }];
+
+  // Find objects in the other dictionary that don't exist in self
+  [b enumerateKeysAndObjectsUsingBlock:^(id _Nonnull key, id _Nonnull obj, BOOL *_Nonnull stop) {
+    id aObj = a[key];
+
+    if (!aObj) {
+      delta[key] = obj;
+    }
+  }];
+
+  return delta;
+}
+
 void SerializeAndCheck(es_event_type_t eventType,
                        void (^messageSetup)(std::shared_ptr<MockEndpointSecurityAPI>,
                                             es_message_t *),
-                       SNTDecisionCache *decisionCache) {
+                       SNTDecisionCache *decisionCache, bool json = false) {
   std::shared_ptr<MockEndpointSecurityAPI> mockESApi = std::make_shared<MockEndpointSecurityAPI>();
 
   for (uint32_t cur_version = 1; cur_version <= MaxSupportedESMessageVersionForCurrentOS();
@@ -185,7 +210,7 @@ void SerializeAndCheck(es_event_type_t eventType,
 
     messageSetup(mockESApi, &esMsg);
 
-    std::shared_ptr<Serializer> bs = Protobuf::Create(mockESApi, decisionCache);
+    std::shared_ptr<Serializer> bs = Protobuf::Create(mockESApi, decisionCache, json);
     std::unique_ptr<EnrichedMessage> enrichedMsg = Enricher().Enrich(Message(mockESApi, &esMsg));
 
     // Copy some values we need to check later before the object is moved out of this funciton
@@ -204,14 +229,43 @@ void SerializeAndCheck(es_event_type_t eventType,
     std::vector<uint8_t> vec = bs->SerializeMessage(std::move(enrichedMsg));
     std::string protoStr(vec.begin(), vec.end());
 
+    // if we're checking against JSON then we should already have a jsonified string and just need
+    // to
     ::pbv1::SantaMessage santaMsg;
-    XCTAssertTrue(santaMsg.ParseFromString(protoStr));
-
-    std::string gotData = ConvertMessageToJsonString(santaMsg);
+    std::string gotData;
+
+    if (json) {
+      // Parse the jsonified string into the protobuf
+      // gotData = protoStr;
+      google::protobuf::util::JsonParseOptions options;
+      options.ignore_unknown_fields = true;
+      google::protobuf::util::Status status = JsonStringToMessage(protoStr, &santaMsg, options);
+      gotData = ConvertMessageToJsonString(santaMsg);
+    } else {
+      XCTAssertTrue(santaMsg.ParseFromString(protoStr));
+      gotData = ConvertMessageToJsonString(santaMsg);
+    }
 
     XCTAssertTrue(CompareTime(santaMsg.processed_time(), enrichmentTime));
     XCTAssertTrue(CompareTime(santaMsg.event_time(), msgTime));
-    XCTAssertEqualObjects([NSString stringWithUTF8String:gotData.c_str()], wantData);
+
+    // Convert JSON strings to objects and compare each key-value set.
+    NSError *jsonError;
+    NSData *objectData = [wantData dataUsingEncoding:NSUTF8StringEncoding];
+    NSDictionary *wantJSONDict =
+      [NSJSONSerialization JSONObjectWithData:objectData
+                                      options:NSJSONReadingMutableContainers
+                                        error:&jsonError];
+    XCTAssertNil(jsonError, @"failed to parse want data as JSON");
+    NSDictionary *gotJSONDict = [NSJSONSerialization
+      JSONObjectWithData:[NSData dataWithBytes:gotData.data() length:gotData.length()]
+                 options:NSJSONReadingMutableContainers
+                   error:&jsonError];
+    XCTAssertNil(jsonError, @"failed to parse got data as JSON");
+
+    // XCTAssertEqualObjects([NSString stringWithUTF8String:gotData.c_str()], wantData);
+    NSDictionary *delta = findDelta(wantJSONDict, gotJSONDict);
+    XCTAssertEqualObjects(@{}, delta);
   }
 
   XCTBubbleMockVerifyAndClearExpectations(mockESApi.get());
@@ -294,8 +348,9 @@ - (void)tearDown {
 
 - (void)serializeAndCheckEvent:(es_event_type_t)eventType
                   messageSetup:(void (^)(std::shared_ptr<MockEndpointSecurityAPI>,
-                                         es_message_t *))messageSetup {
-  SerializeAndCheck(eventType, messageSetup, self.mockDecisionCache);
+                                         es_message_t *))messageSetup
+                          json:(BOOL)json {
+  SerializeAndCheck(eventType, messageSetup, self.mockDecisionCache, (bool)json);
 }
 
 - (void)testSerializeMessageClose {
@@ -306,7 +361,8 @@ - (void)testSerializeMessageClose {
                                  es_message_t *esMsg) {
                     esMsg->event.close.modified = true;
                     esMsg->event.close.target = &file;
-                  }];
+                  }
+                          json:NO];
 }
 
 - (void)testSerializeMessageExchange {
@@ -318,7 +374,8 @@ - (void)testSerializeMessageExchange {
                                  es_message_t *esMsg) {
                     esMsg->event.exchangedata.file1 = &file1;
                     esMsg->event.exchangedata.file2 = &file2;
-                  }];
+                  }
+                          json:NO];
 }
 
 - (void)testGetDecisionEnum {
@@ -455,15 +512,69 @@ - (void)testSerializeMessageExec {
                         .WillOnce(testing::Return(&fd2))
                         .WillOnce(testing::Return(&fd3));
                     }
-                  }];
+                  }
+                          json:NO];
+}
+
+- (void)testSerializeMessageExecJSON {
+  es_file_t procFileTarget = MakeESFile("fooexec", MakeStat(300));
+  __block es_process_t procTarget =
+    MakeESProcess(&procFileTarget, MakeAuditToken(23, 45), MakeAuditToken(67, 89));
+  __block es_file_t fileCwd = MakeESFile("cwd", MakeStat(400));
+  __block es_file_t fileScript = MakeESFile("script.sh", MakeStat(500));
+  __block es_fd_t fd1 = {.fd = 1, .fdtype = PROX_FDTYPE_VNODE};
+  __block es_fd_t fd2 = {.fd = 2, .fdtype = PROX_FDTYPE_SOCKET};
+  __block es_fd_t fd3 = {.fd = 3, .fdtype = PROX_FDTYPE_PIPE, .pipe = {.pipe_id = 123}};
+
+  procTarget.codesigning_flags = CS_SIGNED | CS_HARD | CS_KILL;
+  memset(procTarget.cdhash, 'A', sizeof(procTarget.cdhash));
+  procTarget.signing_id = MakeESStringToken("my_signing_id");
+  procTarget.team_id = MakeESStringToken("my_team_id");
+
+  [self serializeAndCheckEvent:ES_EVENT_TYPE_NOTIFY_EXEC
+                  messageSetup:^(std::shared_ptr<MockEndpointSecurityAPI> mockESApi,
+                                 es_message_t *esMsg) {
+                    esMsg->event.exec.target = &procTarget;
+                    esMsg->event.exec.cwd = &fileCwd;
+                    esMsg->event.exec.script = &fileScript;
+
+                    // For version 5, simulate a "truncated" set of FDs
+                    if (esMsg->version == 5) {
+                      esMsg->event.exec.last_fd = 123;
+                    } else {
+                      esMsg->event.exec.last_fd = 3;
+                    }
+
+                    EXPECT_CALL(*mockESApi, ExecArgCount).WillOnce(testing::Return(3));
+                    EXPECT_CALL(*mockESApi, ExecArg)
+                      .WillOnce(testing::Return(MakeESStringToken("exec_path")))
+                      .WillOnce(testing::Return(MakeESStringToken("-l")))
+                      .WillOnce(testing::Return(MakeESStringToken("--foo")));
+
+                    EXPECT_CALL(*mockESApi, ExecEnvCount).WillOnce(testing::Return(2));
+                    EXPECT_CALL(*mockESApi, ExecEnv)
+                      .WillOnce(
+                        testing::Return(MakeESStringToken("ENV_PATH=/path/to/bin:/and/another")))
+                      .WillOnce(testing::Return(MakeESStringToken("DEBUG=1")));
+
+                    if (esMsg->version >= 4) {
+                      EXPECT_CALL(*mockESApi, ExecFDCount).WillOnce(testing::Return(3));
+                      EXPECT_CALL(*mockESApi, ExecFD)
+                        .WillOnce(testing::Return(&fd1))
+                        .WillOnce(testing::Return(&fd2))
+                        .WillOnce(testing::Return(&fd3));
+                    }
+                  }
+                          json:YES];
 }
 
 - (void)testSerializeMessageExit {
   [self serializeAndCheckEvent:ES_EVENT_TYPE_NOTIFY_EXIT
                   messageSetup:^(std::shared_ptr<MockEndpointSecurityAPI> mockESApi,
                                  es_message_t *esMsg) {
                     esMsg->event.exit.stat = W_EXITCODE(1, 0);
-                  }];
+                  }
+                          json:NO];
 }
 
 - (void)testEncodeExitStatus {
@@ -500,7 +611,8 @@ - (void)testSerializeMessageFork {
                   messageSetup:^(std::shared_ptr<MockEndpointSecurityAPI> mockESApi,
                                  es_message_t *esMsg) {
                     esMsg->event.fork.child = &procChild;
-                  }];
+                  }
+                          json:NO];
 }
 
 - (void)testSerializeMessageLink {
@@ -514,7 +626,8 @@ - (void)testSerializeMessageLink {
                     esMsg->event.link.source = &fileSource;
                     esMsg->event.link.target_dir = &fileTargetDir;
                     esMsg->event.link.target_filename = targetTok;
-                  }];
+                  }
+                          json:NO];
 }
 
 - (void)testSerializeMessageRename {
@@ -535,7 +648,8 @@ - (void)testSerializeMessageRename {
                       esMsg->event.rename.destination.new_path.filename = targetTok;
                       esMsg->event.rename.destination_type = ES_DESTINATION_TYPE_NEW_PATH;
                     }
-                  }];
+                  }
+                          json:NO];
 }
 
 - (void)testSerializeMessageUnlink {
@@ -547,7 +661,8 @@ - (void)testSerializeMessageUnlink {
                                  es_message_t *esMsg) {
                     esMsg->event.unlink.target = &fileTarget;
                     esMsg->event.unlink.parent_dir = &fileTargetParent;
-                  }];
+                  }
+                          json:NO];
 }
 
 - (void)testGetAccessType {

docs/deployment/configuration.md

@@ -54,7 +54,8 @@ also known as mobileconfig files, which are in an Apple-specific XML format.
 | MachineOwnerKey                   | String     | The key to use on MachineOwnerPlist.     |
 | MachineIDPlist                    | String     | The path to a plist that contains the MachineOwnerKey / value pair. |
 | MachineIDKey                      | String     | The key to use on MachineIDPlist.        |
-| EventLogType                      | String     | Defines how event logs are stored. Options are 1) syslog: Sent to ASL or ULS (if built with the 10.12 SDK or later). 2) filelog: Sent to a file on disk. Use EventLogPath to specify a path. 3) protobuf (BETA): Sent to file on disk using a maildir-like format. 4) null: Don't output any event logs. Defaults to filelog.      |
+| EventLogType                      | String     | Defines how event logs are stored. Options are 1) syslog: Sent to ULS. 2) filelog: Sent to a file on disk. Use EventLogPath to specify a path. 3) protobuf (BETA): Sent to file on disk using a maildir-like format. 4) json (BETA): Same as file but output is one JSON object per line 5) null: Don't output any event logs. Defaults to filelog.      |
+| EventLogPath                      | String     | If EventLogType is set to filelog or json, EventLogPath will provide the path to save logs. Defaults to /var/db/santa/santa.log. If you change this value ensure you also update com.google.santa.newsyslog.conf with the new path.        |
 | EventLogPath                      | String     | If EventLogType is set to filelog, EventLogPath will provide the path to save logs. Defaults to /var/db/santa/santa.log. If you change this value ensure you also update com.google.santa.newsyslog.conf with the new path.        |
 | SpoolDirectory                    | String     | If EventLogType is set to protobuf, SpoolDirectory will provide the the base directory used to save files according to a maildir-like format. Defaults to /var/db/santa/spool. |
 | SpoolDirectoryFileSizeThresholdKB  | Integer    | If EventLogType is set to protobuf, SpoolDirectoryFileSizeThresholdKB defines the per-file size limit for files stored in the spool directory. Events are buffered in memory until this threshold would be exceeded (or SpoolDirectoryEventMaxFlushTimeSec is exceeded). Defaults to 100. |