Skip to content

Commit

Permalink
Get rid of unnecessary check in SplitLines iterator (#5141)
Browse files Browse the repository at this point in the history
  • Loading branch information
dannyvankooten authored Oct 8, 2022
1 parent 798882d commit cd46de7
Showing 1 changed file with 11 additions and 1 deletion.
12 changes: 11 additions & 1 deletion polars/polars-io/src/csv/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -243,14 +243,15 @@ impl<'a> Iterator for SplitLines<'a> {
match iter.next() {
Some(&c) => {
pos += 1;

if c == b'"' {
// toggle between string field enclosure
// if we encounter a starting '"' -> in_field = true;
// if we encounter a closing '"' -> in_field = false;
in_field = !in_field;
}
// if we are not in a string and we encounter '\n' we can stop at this position.
if c == self.end_line_char && !in_field {
else if c == self.end_line_char && !in_field {
break;
}
}
Expand Down Expand Up @@ -636,4 +637,13 @@ mod test {
assert_eq!(fields2.next(), Some(("12345".as_bytes(), false)));
assert_eq!(fields2.next(), None);
}

#[test]
fn test_splitlines() {
let input = "1,\"foo\n\"\n2,\"foo\n\"\n";
let mut lines = SplitLines::new(input.as_bytes(), b'\n');
assert_eq!(lines.next(), Some("1,\"foo\n\"".as_bytes()));
assert_eq!(lines.next(), Some("2,\"foo\n\"".as_bytes()));
assert_eq!(lines.next(), None);
}
}

0 comments on commit cd46de7

Please sign in to comment.