fix truncate of oversized log message (#65)

fix truncate of log message size bteween 32k to 500k
logzio · Jun 27, 2023 · 18a9151 · 18a9151
1 parent 8a05d4a
commit 18a9151
Show file tree

Hide file tree

Showing 9 changed files with 43 additions and 45 deletions.
diff --git a/.github/workflows/git-secrets.yml b/.github/workflows/git-secrets.yml
diff --git a/.pre_commit_config.yaml b/.pre_commit_config.yaml
@@ -0,0 +1,8 @@
+repos:
+  - repo: https://github.com/cycodehq-public/cycode-cli
+    rev: 0.1.6
+    hooks:
+      - id: cycode
+        language_version: python3
+        stages:
+          - commit
diff --git a/logzio-sender-test/pom.xml b/logzio-sender-test/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>logzio-java-sender</artifactId>
         <groupId>io.logz.sender</groupId>
-        <version>0-SNAPSHOT</version>
+        <version>1.1.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/logzio-sender/pom.xml b/logzio-sender/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>logzio-java-sender</artifactId>
         <groupId>io.logz.sender</groupId>
-        <version>0-SNAPSHOT</version>
+        <version>1.1.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java b/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java
@@ -144,8 +144,8 @@ public void send(JsonObject jsonMessage) {
 
         // check for oversized message
         int jsonByteLength = jsonMessage.toString().getBytes(StandardCharsets.UTF_8).length;
-        if (jsonByteLength > MAX_LOG_SIZE_IN_BYTES) {
-            String jsonMessageField = jsonMessage.get("message").getAsString();
+        String jsonMessageField = jsonMessage.get("message").getAsString();
+        if (jsonByteLength > MAX_LOG_SIZE_IN_BYTES || jsonMessageField.length() >= MAX_LOG_LINE_SIZE_IN_BYTES) {
 
             // calculate the minimum between max log line size, and the message field after truncating the exceeding bytes
             int truncatedMessageSize = Math.min(MAX_LOG_LINE_SIZE_IN_BYTES - TRUNCATED_MESSAGE_SUFFIX.length(),
@@ -227,7 +227,7 @@ private List<FormattedLogMessage> dequeueUpToMaxBatchSize() {
         return logsList;
     }
 
-    private void drainQueue()   {
+    private void drainQueue() {
         debug("Attempting to drain queue");
         if (!logsQueue.isEmpty()) {
             while (!logsQueue.isEmpty()) {
@@ -240,7 +240,7 @@ private void drainQueue()   {
 
                     // And lets return everything to the queue
                     logsList.forEach((logMessage) -> {
-                            logsQueue.enqueue(logMessage.getMessage());
+                        logsQueue.enqueue(logMessage.getMessage());
                     });
 
                     // Lets wait for a new interval, something is wrong in the server side

diff --git a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java
@@ -24,7 +24,8 @@ public abstract class LogzioSenderTest {
     protected MockLogzioBulkListener mockListener;
     private final static int MAX_LOG_LINE_SIZE_IN_BYTES = 32700;
     private final static String TRUNCATED_MESSAGE_SUFFIX = "...truncated";
-    private final static String EXCEEDING_MESSAGE_FILE_PATH = "src/test/resources/exceeding_max_size_message.log";
+    private final static String EXCEEDING_LOG_FILE_PATH = "src/test/resources/exceeding_max_size_log.log";
+    private final static String EXCEEDING_MESSAGE_FIELD_FILE_PATH = "src/test/resources/exceeding_max_size_message.log";
     private final static Logger logger = LoggerFactory.getLogger(LogzioSenderTest.class);
     private static final int INITIAL_WAIT_BEFORE_RETRY_MS = 2000;
     private static final int MAX_RETRIES_ATTEMPTS = 3;
@@ -396,7 +397,27 @@ public void checkExceedingMaxSizeJsonLogWithCut() throws LogzioParameterErrorExc
         String loggerName = "checkExceedingMaxSizeJsonLogWithCutName";
         int drainTimeout = 2;
 
-        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_MESSAGE_FILE_PATH)), StandardCharsets.UTF_8);
+        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_LOG_FILE_PATH)), StandardCharsets.UTF_8);
+        JsonObject log = createJsonMessage(loggerName, message);
+
+        int logSize = log.toString().getBytes(StandardCharsets.UTF_8).length;
+        ScheduledExecutorService tasks = Executors.newScheduledThreadPool(1);
+        LogzioSender testSender = getLogzioSenderWithAndExceedMaxSizeAction(token, type, drainTimeout, logSize, tasks,"cut");
+
+        testSender.send(log);
+        sleepSeconds(2 * drainTimeout);
+        mockListener.assertLogReceivedByMessage(message.substring(0, MAX_LOG_LINE_SIZE_IN_BYTES - TRUNCATED_MESSAGE_SUFFIX.length()) + TRUNCATED_MESSAGE_SUFFIX);
+        tasks.shutdownNow();
+    }
+
+    @Test
+    public void checkExceedingMaxSizeJsonMessagFieldeWithCut() throws LogzioParameterErrorException, IOException {
+        String token = "checkExceedingMaxSizeJsonLogWithCut";
+        String type = random(8);
+        String loggerName = "checkExceedingMaxSizeJsonLogWithCutName";
+        int drainTimeout = 2;
+
+        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_MESSAGE_FIELD_FILE_PATH)), StandardCharsets.UTF_8);
         JsonObject log = createJsonMessage(loggerName, message);
 
         int logSize = log.toString().getBytes(StandardCharsets.UTF_8).length;
@@ -416,7 +437,7 @@ public void checkExceedingMaxSizeBytesLogWithCut() throws LogzioParameterErrorEx
         String loggerName = "checkExceedingMaxSizeBytesLogWithCutName";
         int drainTimeout = 2;
 
-        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_MESSAGE_FILE_PATH)), StandardCharsets.UTF_8);
+        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_LOG_FILE_PATH)), StandardCharsets.UTF_8);
         JsonObject log = createJsonMessage(loggerName, message);
 
         int logSize = log.toString().getBytes(StandardCharsets.UTF_8).length;
@@ -437,7 +458,7 @@ public void checkExceedingMaxSizeJsonLogWithDrop() throws LogzioParameterErrorEx
         String loggerName = "checkExceedingMaxSizeJsonLogWithDropName";
         int drainTimeout = 2;
 
-        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_MESSAGE_FILE_PATH)), StandardCharsets.UTF_8);
+        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_LOG_FILE_PATH)), StandardCharsets.UTF_8);
         JsonObject log = createJsonMessage(loggerName, message);
 
         int logSize = log.toString().getBytes(StandardCharsets.UTF_8).length;
@@ -456,7 +477,7 @@ public void checkExceedingMaxSizeBytesLogWithDrop() throws LogzioParameterErrorE
         String loggerName = "checkExceedingMaxSizeBytesLogWithDropName";
         int drainTimeout = 2;
 
-        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_MESSAGE_FILE_PATH)), StandardCharsets.UTF_8);
+        String message = new String(Files.readAllBytes(Paths.get(EXCEEDING_LOG_FILE_PATH)), StandardCharsets.UTF_8);
         JsonObject log = createJsonMessage(loggerName, message);
 
         int logSize = log.toString().getBytes(StandardCharsets.UTF_8).length;

diff --git a/logzio-sender/src/test/resources/exceeding_max_size_log.log b/logzio-sender/src/test/resources/exceeding_max_size_log.log
diff --git a/logzio-sender/src/test/resources/exceeding_max_size_message.log b/logzio-sender/src/test/resources/exceeding_max_size_message.log
diff --git a/pom.xml b/pom.xml
@@ -7,7 +7,7 @@
     <groupId>io.logz.sender</groupId>
     <artifactId>logzio-java-sender</artifactId>
     <packaging>pom</packaging>
-    <version>0-SNAPSHOT</version>
+    <version>1.1.8</version>
 
     <name>Logz.io Logs Sender</name>
     <description>Send your log messages to your logz.io account in an encrypted, non-blocking manner.</description>