Fix split with delimiter (#32298)

apache · Aug 23, 2024 · 46cdc4b · 46cdc4b
1 parent e24a0c4
commit 46cdc4b
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java
@@ -438,7 +438,11 @@ private boolean readCustomLine() throws IOException {
                 // Add to str any previous partial delimiter since we didn't match the whole
                 // delimiter
                 str.write(delimiter, 0, prevDelPosn);
-                delPosn = 0;
+                if (buffer[bufferPosn] == delimiter[0]) {
+                  delPosn = 1;
+                } else {
+                  delPosn = 0;
+                }
                 break; // Leave this loop and use the fast-path delimiter matching
               }
             }
@@ -452,6 +456,8 @@ private boolean readCustomLine() throws IOException {
                 bufferPosn++;
                 break;
               }
+            } else if (buffer[bufferPosn] == delimiter[0]) {
+              delPosn = 1;
             } else {
               delPosn = 0;
             }

diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
@@ -674,8 +674,10 @@ public void testReadStringsWithCustomDelimiter() throws Exception {
             "To be, or not to be: that *is the question: ",
             // complete delimiter
             "Whether 'tis nobler in the mind to suffer |*",
+            // edge case: partial delimiter then complete delimiter
+            "The slings and arrows of outrageous fortune,*||**|",
             // truncated delimiter
-            "The slings and arrows of outrageous fortune,|"
+            "Or to take arms against a sea of troubles,|"
           };
 
       File tmpFile = tempFolder.newFile("tmpfile.txt");
@@ -689,7 +691,8 @@ public void testReadStringsWithCustomDelimiter() throws Exception {
           .containsInAnyOrder(
               "To be, or not to be: that |is the question: To be, or not to be: "
                   + "that *is the question: Whether 'tis nobler in the mind to suffer ",
-              "The slings and arrows of outrageous fortune,|");
+              "The slings and arrows of outrageous fortune,*|",
+              "*|Or to take arms against a sea of troubles,|");
       p.run();
     }