Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow metadata for write_deltalake #587

Merged
merged 4 commits into from
Apr 17, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion python/deltalake/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ def write_deltalake(
partition_by: Optional[List[str]] = None,
filesystem: Optional[pa_fs.FileSystem] = None,
mode: Literal["error", "append", "overwrite", "ignore"] = "error",
name: Optional[str] = None,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we also add a note in the docstring that this function does not register this table in your data catalog? Users will have to either use the filesystem path to the table or manually register this table in the catalog. I'll create a follow up issue for us to handle that though.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a brief note. Let me know if more explanation re: manual add is needed.

description: Optional[str] = None,
configuration: Optional[Mapping[str, Optional[str]]] = None,
PadenZach marked this conversation as resolved.
Show resolved Hide resolved
) -> None:
"""Write to a Delta Lake table (Experimental)

Expand All @@ -53,6 +56,8 @@ def write_deltalake(
to write to an existing table with a higher min_writer_version, this
function will throw DeltaTableProtocolError.

Note that this function does NOT register this table in a data catalog.

:param table_or_uri: URI of a table or a DeltaTable object.
:param data: Data to write. If passing iterable, the schema must also be given.
:param schema: Optional schema to write.
Expand All @@ -64,6 +69,9 @@ def write_deltalake(
already exists. If 'append', will add new data. If 'overwrite', will
replace table with new data. If 'ignore', will not write anything if
table already exists.
:param name: User-provided identifier for this table.
:param description: User-provided description for this table.
:param configuration: A map containing configuration options for the metadata action.
"""
if isinstance(data, pd.DataFrame):
data = pa.Table.from_pandas(data)
Expand Down Expand Up @@ -147,7 +155,16 @@ def visitor(written_file: Any) -> None:
)

if table is None:
_write_new_deltalake(table_uri, schema, add_actions, mode, partition_by or [])
_write_new_deltalake( # type: ignore[call-arg]
table_uri,
schema,
add_actions,
mode,
partition_by or [],
name,
description,
configuration,
)
else:
table._table.create_write_transaction(
add_actions,
Expand Down
12 changes: 8 additions & 4 deletions python/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -514,12 +514,16 @@ impl From<&PyAddAction> for action::Add {
}

#[pyfunction]
#[allow(clippy::too_many_arguments)]
fn write_new_deltalake(
table_uri: String,
schema: ArrowSchema,
add_actions: Vec<PyAddAction>,
_mode: &str,
partition_by: Vec<String>,
name: Option<String>,
description: Option<String>,
configuration: Option<HashMap<String, Option<String>>>,
) -> PyResult<()> {
let mut table = deltalake::DeltaTable::new(
&table_uri,
Expand All @@ -529,12 +533,12 @@ fn write_new_deltalake(
.map_err(PyDeltaTableError::from_raw)?;

let metadata = DeltaTableMetaData::new(
None,
None,
None,
name,
description,
None, // Format
(&schema).try_into()?,
partition_by,
HashMap::new(),
configuration.unwrap_or_default(),
);

let fut = table.create(
Expand Down
18 changes: 18 additions & 0 deletions python/tests/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,24 @@ def test_roundtrip_basic(tmp_path: pathlib.Path, sample_data: pa.Table):
assert table == sample_data


def test_roundtrip_metadata(tmp_path: pathlib.Path, sample_data: pa.Table):
write_deltalake(
str(tmp_path),
sample_data,
name="test_name",
description="test_desc",
configuration={"configTest": "foobar"},
)

delta_table = DeltaTable(str(tmp_path))

metadata = delta_table.metadata()

assert metadata.name == "test_name"
assert metadata.description == "test_desc"
assert metadata.configuration == {"configTest": "foobar"}


@pytest.mark.parametrize(
"column",
[
Expand Down