Skip to content

Commit

Permalink
fix(#2143): Keep specific error type when writing fails.
Browse files Browse the repository at this point in the history
  • Loading branch information
abaerptc authored and rtyler committed Jan 29, 2024
1 parent 857bd0e commit dcbfb62
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 0 deletions.
33 changes: 33 additions & 0 deletions crates/core/src/writer/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ fn extract_partition_values(

#[cfg(test)]
mod tests {
use arrow_schema::ArrowError;
use parquet::file::reader::FileReader;
use parquet::file::serialized_reader::SerializedFileReader;
use std::fs::File;
Expand Down Expand Up @@ -510,4 +511,36 @@ mod tests {
);
assert!(extract_partition_values(&[String::from("col4")], &record_batch).is_err())
}

#[tokio::test]
async fn test_parsing_error() {
let table_dir = tempfile::tempdir().unwrap();
let schema = get_delta_schema();
let path = table_dir.path().to_str().unwrap().to_string();

let arrow_schema = <ArrowSchema as TryFrom<&StructType>>::try_from(&schema).unwrap();
let mut writer = JsonWriter::try_new(
path.clone(),
Arc::new(arrow_schema),
Some(vec!["modified".to_string()]),
None,
)
.unwrap();

let data = serde_json::json!(
{
"id" : "A",
"value": "abc",
"modified": "2021-02-01"
}
);

let res = writer.write(vec![data]).await;
assert!(matches!(
res,
Err(DeltaTableError::Arrow {
source: ArrowError::JsonError(_)
})
));
}
}
1 change: 1 addition & 0 deletions crates/core/src/writer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ impl From<DeltaWriterError> for DeltaTableError {
DeltaWriterError::Io { source } => DeltaTableError::Io { source },
DeltaWriterError::ObjectStore { source } => DeltaTableError::ObjectStore { source },
DeltaWriterError::Parquet { source } => DeltaTableError::Parquet { source },
DeltaWriterError::DeltaTable(e) => e,
_ => DeltaTableError::Generic(err.to_string()),
}
}
Expand Down

0 comments on commit dcbfb62

Please sign in to comment.