Skip to content

Commit

Permalink
FIX: Removes the csv-sniffer in favour of a simple guess
Browse files Browse the repository at this point in the history
This removes 500Kb form the was build so it is worth it.
We were using a very old version of the sniffer, the last one might not
have this bug though
  • Loading branch information
nhatcher committed Nov 26, 2024
1 parent 1f1fd24 commit 949eafc
Show file tree
Hide file tree
Showing 5 changed files with 63 additions and 93 deletions.
90 changes: 6 additions & 84 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion base/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ regex = "1.0"
once_cell = "1.16.0"
bitcode = "0.6.0"
csv = "1.3.0"
csv-sniffer = "0.1"

[dev-dependencies]
serde_json = "1.0"
Expand Down
21 changes: 21 additions & 0 deletions base/src/test/user_model/test_paste_csv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,27 @@ fn csv_paste() {
);
}

#[test]
fn csv_paste_formula() {
let mut model = UserModel::new_empty("model", "en", "UTC").unwrap();

let csv = "=YEAR(TODAY())";
let area = Area {
sheet: 0,
row: 1,
column: 1,
width: 1,
height: 1,
};
model.set_selected_cell(1, 1).unwrap();
model.paste_csv_string(&area, csv).unwrap();

assert_eq!(
model.get_formatted_cell_value(0, 1, 1),
Ok("2022".to_string())
);
}

#[test]
fn tsv_crlf_paste() {
let mut model = UserModel::new_empty("model", "en", "UTC").unwrap();
Expand Down
43 changes: 35 additions & 8 deletions base/src/user_model/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
use std::{collections::HashMap, fmt::Debug, io::Cursor};

use csv::{ReaderBuilder, WriterBuilder};
use csv_sniffer::Sniffer;
use serde::{Deserialize, Serialize};

use crate::{
Expand Down Expand Up @@ -62,6 +61,30 @@ pub struct BorderArea {
r#type: BorderType,
}

fn guess_delimiter(data: &str) -> char {
let delimiters = [',', ';', '\t', '|', ':'];
let mut best_delim = ',';
let mut max_fields = 0;

for &delim in &delimiters {
let mut fields_per_line = Vec::new();

for line in data.lines() {
let fields = line.split(delim).count();
fields_per_line.push(fields);
}

let first_count = fields_per_line.first().copied().unwrap_or(0);

if fields_per_line.iter().all(|&count| count == first_count) && first_count > max_fields {
max_fields = first_count;
best_delim = delim;
}
}

best_delim
}

fn boolean(value: &str) -> Result<bool, String> {
match value {
"true" => Ok(true),
Expand Down Expand Up @@ -1509,18 +1532,13 @@ impl UserModel {
let sheet = area.sheet;
let mut row = area.row;
let mut column = area.column;
// Create a sniffer with default settings
let mut sniffer = Sniffer::new();
let mut csv_reader = Cursor::new(csv);

// Sniff the CSV metadata
let metadata = sniffer
.sniff_reader(&mut csv_reader)
.map_err(|_| "Failed")?;
let delimiter = guess_delimiter(csv) as u8;
// Reset the cursor to the beginning after sniffing
csv_reader.set_position(0);
let mut reader = ReaderBuilder::new()
.delimiter(metadata.dialect.delimiter)
.delimiter(delimiter)
.has_headers(false)
.from_reader(csv_reader);
for record in reader.records() {
Expand Down Expand Up @@ -1876,6 +1894,8 @@ mod tests {
user_model::common::{horizontal, vertical},
};

use super::guess_delimiter;

#[test]
fn test_vertical() {
let all = vec![
Expand Down Expand Up @@ -1906,4 +1926,11 @@ mod tests {
assert_eq!(horizontal(&format!("{}", a)), Ok(a));
}
}

#[test]
fn test_guess_delimiter() {
assert_eq!(guess_delimiter("1,2,3\n4,5,6"), ',');
assert_eq!(guess_delimiter("1\t2\t3\n4\t5\t6"), '\t');
assert_eq!(guess_delimiter("1"), ',');
}
}
1 change: 1 addition & 0 deletions webapp/src/components/editor/editor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ const Editor = (options: EditorOptions) => {
maskRef.current.style.top = `-${textareaRef.current.scrollTop}px`;
}
}}
onPaste={(event) => event.stopPropagation()}
/>
</div>
);
Expand Down

0 comments on commit 949eafc

Please sign in to comment.