diff --git a/crates/aws/Cargo.toml b/crates/aws/Cargo.toml index b0c61bb38d..992a32c93e 100644 --- a/crates/aws/Cargo.toml +++ b/crates/aws/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-aws" -version = "0.2.1" +version = "0.3.0" authors.workspace = true keywords.workspace = true readme.workspace = true @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.1", path = "../core" } +deltalake-core = { version = "0.20.0", path = "../core" } aws-smithy-runtime-api = { version="1.7" } aws-smithy-runtime = { version="1.7", optional = true} aws-credential-types = { version="1.2", features = ["hardcoded-credentials"]} diff --git a/crates/azure/Cargo.toml b/crates/azure/Cargo.toml index 56181a9f0c..87a744d608 100644 --- a/crates/azure/Cargo.toml +++ b/crates/azure/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-azure" -version = "0.2.0" +version = "0.3.0" authors.workspace = true keywords.workspace = true readme.workspace = true @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.1", path = "../core" } +deltalake-core = { version = "0.20.0", path = "../core" } lazy_static = "1" # workspace depenndecies diff --git a/crates/catalog-glue/Cargo.toml b/crates/catalog-glue/Cargo.toml index b107d1955a..549b3a11c8 100644 --- a/crates/catalog-glue/Cargo.toml +++ b/crates/catalog-glue/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-catalog-glue" -version = "0.3.0" +version = "0.4.0" authors.workspace = true keywords.workspace = true readme.workspace = true @@ -15,7 +15,7 @@ rust-version.workspace = true async-trait = { workspace = true } aws-config = "1" aws-sdk-glue = "1" -deltalake-core = { version = "0.19.1", path = "../core" } +deltalake-core = { version = "0.20.0", path = "../core" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 5d91c9f61d..52df035c71 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-core" -version = "0.19.1" +version = "0.20.0" authors.workspace = true keywords.workspace = true readme.workspace = true diff --git a/crates/core/src/kernel/models/actions.rs b/crates/core/src/kernel/models/actions.rs index 739a585509..6ec8fc11fb 100644 --- a/crates/core/src/kernel/models/actions.rs +++ b/crates/core/src/kernel/models/actions.rs @@ -9,7 +9,7 @@ use url::Url; use super::schema::StructType; use crate::kernel::{error::Error, DeltaResult}; -use crate::DeltaConfigKey; +use crate::TableProperty; /// Defines a file format used in table #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -240,10 +240,10 @@ impl Protocol { new_properties: &HashMap, raise_if_not_exists: bool, ) -> DeltaResult { - let mut parsed_properties: HashMap = HashMap::new(); + let mut parsed_properties: HashMap = HashMap::new(); for (key, value) in new_properties { - if let Ok(parsed_key) = key.parse::() { + if let Ok(parsed_key) = key.parse::() { parsed_properties.insert(parsed_key, value.to_string()); } else if raise_if_not_exists { return Err(Error::Generic(format!( @@ -254,7 +254,7 @@ impl Protocol { } // Check and update delta.minReaderVersion - if let Some(min_reader_version) = parsed_properties.get(&DeltaConfigKey::MinReaderVersion) { + if let Some(min_reader_version) = parsed_properties.get(&TableProperty::MinReaderVersion) { let new_min_reader_version = min_reader_version.parse::(); match new_min_reader_version { Ok(version) => match version { @@ -280,7 +280,7 @@ impl Protocol { } // Check and update delta.minWriterVersion - if let Some(min_writer_version) = parsed_properties.get(&DeltaConfigKey::MinWriterVersion) { + if let Some(min_writer_version) = parsed_properties.get(&TableProperty::MinWriterVersion) { let new_min_writer_version = min_writer_version.parse::(); match new_min_writer_version { Ok(version) => match version { @@ -306,7 +306,7 @@ impl Protocol { } // Check enableChangeDataFeed and bump protocol or add writerFeature if writer versions is >=7 - if let Some(enable_cdf) = parsed_properties.get(&DeltaConfigKey::EnableChangeDataFeed) { + if let Some(enable_cdf) = parsed_properties.get(&TableProperty::EnableChangeDataFeed) { let if_enable_cdf = enable_cdf.to_ascii_lowercase().parse::(); match if_enable_cdf { Ok(true) => { @@ -335,7 +335,7 @@ impl Protocol { } } - if let Some(enable_dv) = parsed_properties.get(&DeltaConfigKey::EnableDeletionVectors) { + if let Some(enable_dv) = parsed_properties.get(&TableProperty::EnableDeletionVectors) { let if_enable_dv = enable_dv.to_ascii_lowercase().parse::(); match if_enable_dv { Ok(true) => { diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 86a381de70..0498e4493c 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -93,7 +93,7 @@ pub use self::errors::*; pub use self::schema::partitions::*; pub use self::schema::*; pub use self::table::builder::{DeltaTableBuilder, DeltaTableConfig, DeltaVersion}; -pub use self::table::config::DeltaConfigKey; +pub use self::table::config::TableProperty; pub use self::table::DeltaTable; pub use object_store::{path::Path, Error as ObjectStoreError, ObjectMeta, ObjectStore}; pub use operations::DeltaOps; diff --git a/crates/core/src/operations/cdc.rs b/crates/core/src/operations/cdc.rs index 6e4c9fa50d..b04c794c61 100644 --- a/crates/core/src/operations/cdc.rs +++ b/crates/core/src/operations/cdc.rs @@ -88,7 +88,7 @@ mod tests { use crate::kernel::DataType as DeltaDataType; use crate::kernel::{Action, PrimitiveType, Protocol}; use crate::operations::DeltaOps; - use crate::{DeltaConfigKey, DeltaTable}; + use crate::{DeltaTable, TableProperty}; use arrow::array::{ArrayRef, Int32Array, StructArray}; use arrow::datatypes::{DataType, Field}; use arrow_array::RecordBatch; @@ -130,7 +130,7 @@ mod tests { None, ) .with_actions(actions) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .expect("failed to make a version 4 table with EnableChangeDataFeed"); table.load().await.expect("Failed to reload table"); @@ -185,7 +185,7 @@ mod tests { None, ) .with_actions(actions) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .expect("failed to make a version 4 table with EnableChangeDataFeed"); table.load().await.expect("Failed to reload table"); diff --git a/crates/core/src/operations/convert_to_delta.rs b/crates/core/src/operations/convert_to_delta.rs index ceb6cac3c4..148b581d8b 100644 --- a/crates/core/src/operations/convert_to_delta.rs +++ b/crates/core/src/operations/convert_to_delta.rs @@ -23,7 +23,7 @@ use crate::{ operations::create::CreateBuilder, protocol::SaveMode, table::builder::ensure_table_uri, - table::config::DeltaConfigKey, + table::config::TableProperty, writer::stats::stats_from_parquet_metadata, DeltaResult, DeltaTable, DeltaTableError, ObjectStoreError, NULL_PARTITION_VALUE_DATA_PATH, }; @@ -212,7 +212,7 @@ impl ConvertToDeltaBuilder { /// Specify a table property in the table configuration pub fn with_configuration_property( mut self, - key: DeltaConfigKey, + key: TableProperty, value: Option>, ) -> Self { self.configuration diff --git a/crates/core/src/operations/create.rs b/crates/core/src/operations/create.rs index 63b6995f9b..5e26d7f1fb 100644 --- a/crates/core/src/operations/create.rs +++ b/crates/core/src/operations/create.rs @@ -17,7 +17,7 @@ use crate::kernel::{ use crate::logstore::{LogStore, LogStoreRef}; use crate::protocol::{DeltaOperation, SaveMode}; use crate::table::builder::ensure_table_uri; -use crate::table::config::DeltaConfigKey; +use crate::table::config::TableProperty; use crate::{DeltaTable, DeltaTableBuilder}; #[derive(thiserror::Error, Debug)] @@ -193,7 +193,7 @@ impl CreateBuilder { /// Specify a table property in the table configuration pub fn with_configuration_property( mut self, - key: DeltaConfigKey, + key: TableProperty, value: Option>, ) -> Self { self.configuration @@ -213,7 +213,7 @@ impl CreateBuilder { self } - /// Specify whether to raise an error if the table properties in the configuration are not DeltaConfigKeys + /// Specify whether to raise an error if the table properties in the configuration are not TablePropertys pub fn with_raise_if_key_not_exists(mut self, raise_if_key_not_exists: bool) -> Self { self.raise_if_key_not_exists = raise_if_key_not_exists; self @@ -393,7 +393,7 @@ impl std::future::IntoFuture for CreateBuilder { mod tests { use super::*; use crate::operations::DeltaOps; - use crate::table::config::DeltaConfigKey; + use crate::table::config::TableProperty; use crate::writer::test_utils::{get_delta_schema, get_record_batch}; use tempfile::TempDir; @@ -485,14 +485,14 @@ mod tests { let table = CreateBuilder::new() .with_location("memory://") .with_columns(schema.fields().cloned()) - .with_configuration_property(DeltaConfigKey::AppendOnly, Some("true")) + .with_configuration_property(TableProperty::AppendOnly, Some("true")) .await .unwrap(); let append = table .metadata() .unwrap() .configuration - .get(DeltaConfigKey::AppendOnly.as_ref()) + .get(TableProperty::AppendOnly.as_ref()) .unwrap() .as_ref() .unwrap() diff --git a/crates/core/src/operations/delete.rs b/crates/core/src/operations/delete.rs index f62e8b8130..7dc58b5929 100644 --- a/crates/core/src/operations/delete.rs +++ b/crates/core/src/operations/delete.rs @@ -432,8 +432,8 @@ mod tests { use crate::writer::test_utils::{ get_arrow_schema, get_delta_schema, get_record_batch, setup_table_with_configuration, }; - use crate::DeltaConfigKey; use crate::DeltaTable; + use crate::TableProperty; use arrow::array::Int32Array; use arrow::datatypes::{Field, Schema}; use arrow::record_batch::RecordBatch; @@ -465,7 +465,7 @@ mod tests { #[tokio::test] async fn test_delete_when_delta_table_is_append_only() { - let table = setup_table_with_configuration(DeltaConfigKey::AppendOnly, Some("true")).await; + let table = setup_table_with_configuration(TableProperty::AppendOnly, Some("true")).await; let batch = get_record_batch(None, false); // append some data let table = write_batch(table, batch).await; @@ -909,7 +909,7 @@ mod tests { true, None, ) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); @@ -987,7 +987,7 @@ mod tests { None, ) .with_partition_columns(vec!["year"]) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); diff --git a/crates/core/src/operations/load_cdf.rs b/crates/core/src/operations/load_cdf.rs index eb0d5b1a7b..ad2986de80 100644 --- a/crates/core/src/operations/load_cdf.rs +++ b/crates/core/src/operations/load_cdf.rs @@ -409,7 +409,7 @@ pub(crate) mod tests { use crate::test_utils::TestSchemas; use crate::writer::test_utils::TestResult; - use crate::{DeltaConfigKey, DeltaOps, DeltaTable}; + use crate::{DeltaOps, DeltaTable, TableProperty}; #[tokio::test] async fn test_load_local() -> TestResult { @@ -603,7 +603,7 @@ pub(crate) mod tests { .create() .with_columns(delta_schema.fields().cloned()) .with_partition_columns(["id"]) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); diff --git a/crates/core/src/operations/merge/mod.rs b/crates/core/src/operations/merge/mod.rs index 86930e3a26..b081b387ca 100644 --- a/crates/core/src/operations/merge/mod.rs +++ b/crates/core/src/operations/merge/mod.rs @@ -1386,8 +1386,8 @@ mod tests { use crate::writer::test_utils::get_arrow_schema; use crate::writer::test_utils::get_delta_schema; use crate::writer::test_utils::setup_table_with_configuration; - use crate::DeltaConfigKey; use crate::DeltaTable; + use crate::TableProperty; use arrow::datatypes::Schema as ArrowSchema; use arrow::record_batch::RecordBatch; use arrow_schema::DataType as ArrowDataType; @@ -1426,7 +1426,7 @@ mod tests { #[tokio::test] async fn test_merge_when_delta_table_is_append_only() { let schema = get_arrow_schema(&None); - let table = setup_table_with_configuration(DeltaConfigKey::AppendOnly, Some("true")).await; + let table = setup_table_with_configuration(TableProperty::AppendOnly, Some("true")).await; // append some data let table = write_data(table, &schema).await; // merge @@ -3107,7 +3107,7 @@ mod tests { .create() .with_columns(schema.fields().cloned()) .with_actions(actions) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); @@ -3202,7 +3202,7 @@ mod tests { .create() .with_columns(schema.fields().cloned()) .with_actions(actions) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); diff --git a/crates/core/src/operations/transaction/protocol.rs b/crates/core/src/operations/transaction/protocol.rs index 6e16b95809..b9ea7d65aa 100644 --- a/crates/core/src/operations/transaction/protocol.rs +++ b/crates/core/src/operations/transaction/protocol.rs @@ -272,7 +272,7 @@ mod tests { use crate::protocol::SaveMode; use crate::table::state::DeltaTableState; use crate::test_utils::{ActionFactory, TestSchemas}; - use crate::DeltaConfigKey; + use crate::TableProperty; fn metadata_action(configuration: Option>>) -> Metadata { ActionFactory::metadata(TestSchemas::simple(), None::>, configuration) @@ -328,7 +328,7 @@ mod tests { ..Default::default() }), metadata_action(Some(HashMap::from([( - DeltaConfigKey::AppendOnly.as_ref().to_string(), + TableProperty::AppendOnly.as_ref().to_string(), Some(append.to_string()), )]))) .into(), @@ -630,7 +630,7 @@ mod tests { )])), ) .with_actions(actions) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .expect("failed to make a version 4 table with EnableChangeDataFeed"); let eager_5 = table diff --git a/crates/core/src/operations/update.rs b/crates/core/src/operations/update.rs index bb1262f5c0..61dc4b2f46 100644 --- a/crates/core/src/operations/update.rs +++ b/crates/core/src/operations/update.rs @@ -500,7 +500,7 @@ mod tests { use crate::writer::test_utils::{ get_arrow_schema, get_delta_schema, get_record_batch, setup_table_with_configuration, }; - use crate::{DeltaConfigKey, DeltaTable}; + use crate::{DeltaTable, TableProperty}; use arrow::array::{Int32Array, StringArray}; use arrow::datatypes::Schema as ArrowSchema; use arrow::datatypes::{Field, Schema}; @@ -549,7 +549,7 @@ mod tests { #[tokio::test] async fn test_update_when_delta_table_is_append_only() { - let table = setup_table_with_configuration(DeltaConfigKey::AppendOnly, Some("true")).await; + let table = setup_table_with_configuration(TableProperty::AppendOnly, Some("true")).await; let batch = get_record_batch(None, false); // Append let table = write_batch(table, batch).await; @@ -1062,7 +1062,7 @@ mod tests { None, ) .with_actions(actions) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); @@ -1147,7 +1147,7 @@ mod tests { ) .with_partition_columns(vec!["year"]) .with_actions(actions) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); diff --git a/crates/core/src/operations/write.rs b/crates/core/src/operations/write.rs index cce4ed0901..36dcec5b70 100644 --- a/crates/core/src/operations/write.rs +++ b/crates/core/src/operations/write.rs @@ -1231,7 +1231,7 @@ mod tests { get_arrow_schema, get_delta_schema, get_delta_schema_with_nested_struct, get_record_batch, get_record_batch_with_nested_struct, setup_table_with_configuration, }; - use crate::DeltaConfigKey; + use crate::TableProperty; use arrow_array::{Int32Array, StringArray, TimestampMicrosecondArray}; use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; use datafusion::prelude::*; @@ -1257,7 +1257,7 @@ mod tests { #[tokio::test] async fn test_write_when_delta_table_is_append_only() { - let table = setup_table_with_configuration(DeltaConfigKey::AppendOnly, Some("true")).await; + let table = setup_table_with_configuration(TableProperty::AppendOnly, Some("true")).await; let batch = get_record_batch(None, false); // Append let table = write_batch(table, batch.clone()).await; @@ -2068,7 +2068,7 @@ mod tests { .create() .with_columns(delta_schema.fields().cloned()) .with_partition_columns(["id"]) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); @@ -2141,7 +2141,7 @@ mod tests { .create() .with_columns(delta_schema.fields().cloned()) .with_partition_columns(["id"]) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); @@ -2217,7 +2217,7 @@ mod tests { .create() .with_columns(delta_schema.fields().cloned()) .with_partition_columns(["id"]) - .with_configuration_property(DeltaConfigKey::EnableChangeDataFeed, Some("true")) + .with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true")) .await .unwrap(); assert_eq!(table.version(), 0); diff --git a/crates/core/src/table/config.rs b/crates/core/src/table/config.rs index 68b41d6f67..bc04ec6e91 100644 --- a/crates/core/src/table/config.rs +++ b/crates/core/src/table/config.rs @@ -14,7 +14,7 @@ use crate::errors::DeltaTableError; /// #[derive(PartialEq, Eq, Hash)] #[non_exhaustive] -pub enum DeltaConfigKey { +pub enum TableProperty { /// true for this Delta table to be append-only. If append-only, /// existing records cannot be deleted, and existing values cannot be updated. AppendOnly, @@ -116,7 +116,7 @@ pub enum DeltaConfigKey { CheckpointPolicy, } -impl AsRef for DeltaConfigKey { +impl AsRef for TableProperty { fn as_ref(&self) -> &str { match self { Self::AppendOnly => "delta.appendOnly", @@ -146,7 +146,7 @@ impl AsRef for DeltaConfigKey { } } -impl FromStr for DeltaConfigKey { +impl FromStr for TableProperty { type Err = DeltaTableError; fn from_str(s: &str) -> Result { @@ -217,28 +217,28 @@ impl<'a> TableConfig<'a> { table_config!( ( "true for this Delta table to be append-only", - DeltaConfigKey::AppendOnly, + TableProperty::AppendOnly, append_only, bool, false ), ( "true for Delta Lake to write file statistics in checkpoints in JSON format for the stats column.", - DeltaConfigKey::CheckpointWriteStatsAsJson, + TableProperty::CheckpointWriteStatsAsJson, write_stats_as_json, bool, true ), ( "true for Delta Lake to write file statistics to checkpoints in struct format", - DeltaConfigKey::CheckpointWriteStatsAsStruct, + TableProperty::CheckpointWriteStatsAsStruct, write_stats_as_struct, bool, false ), ( "The target file size in bytes or higher units for file tuning", - DeltaConfigKey::TargetFileSize, + TableProperty::TargetFileSize, target_file_size, i64, // Databricks / spark defaults to 104857600 (bytes) or 100mb @@ -246,14 +246,14 @@ impl<'a> TableConfig<'a> { ), ( "true to enable change data feed.", - DeltaConfigKey::EnableChangeDataFeed, + TableProperty::EnableChangeDataFeed, enable_change_data_feed, bool, false ), ( "true to enable deletion vectors and predictive I/O for updates.", - DeltaConfigKey::EnableDeletionVectors, + TableProperty::EnableDeletionVectors, enable_deletion_vectors, bool, // in databricks the default is dependent on the workspace settings and runtime version @@ -262,21 +262,21 @@ impl<'a> TableConfig<'a> { ), ( "The number of columns for Delta Lake to collect statistics about for data skipping.", - DeltaConfigKey::DataSkippingNumIndexedCols, + TableProperty::DataSkippingNumIndexedCols, num_indexed_cols, i32, 32 ), ( "whether to cleanup expired logs", - DeltaConfigKey::EnableExpiredLogCleanup, + TableProperty::EnableExpiredLogCleanup, enable_expired_log_cleanup, bool, true ), ( "Interval (number of commits) after which a new checkpoint should be created", - DeltaConfigKey::CheckpointInterval, + TableProperty::CheckpointInterval, checkpoint_interval, i32, 100 @@ -297,7 +297,7 @@ impl<'a> TableConfig<'a> { static ref DEFAULT_DURATION: Duration = parse_interval("interval 1 weeks").unwrap(); } self.0 - .get(DeltaConfigKey::DeletedFileRetentionDuration.as_ref()) + .get(TableProperty::DeletedFileRetentionDuration.as_ref()) .and_then(|o| o.as_ref().and_then(|v| parse_interval(v).ok())) .unwrap_or_else(|| DEFAULT_DURATION.to_owned()) } @@ -313,7 +313,7 @@ impl<'a> TableConfig<'a> { static ref DEFAULT_DURATION: Duration = parse_interval("interval 30 days").unwrap(); } self.0 - .get(DeltaConfigKey::LogRetentionDuration.as_ref()) + .get(TableProperty::LogRetentionDuration.as_ref()) .and_then(|o| o.as_ref().and_then(|v| parse_interval(v).ok())) .unwrap_or_else(|| DEFAULT_DURATION.to_owned()) } @@ -323,7 +323,7 @@ impl<'a> TableConfig<'a> { /// Valid values are `Serializable` and `WriteSerializable`. pub fn isolation_level(&self) -> IsolationLevel { self.0 - .get(DeltaConfigKey::IsolationLevel.as_ref()) + .get(TableProperty::IsolationLevel.as_ref()) .and_then(|o| o.as_ref().and_then(|v| v.parse().ok())) .unwrap_or_default() } @@ -331,7 +331,7 @@ impl<'a> TableConfig<'a> { /// Policy applied during chepoint creation pub fn checkpoint_policy(&self) -> CheckpointPolicy { self.0 - .get(DeltaConfigKey::CheckpointPolicy.as_ref()) + .get(TableProperty::CheckpointPolicy.as_ref()) .and_then(|o| o.as_ref().and_then(|v| v.parse().ok())) .unwrap_or_default() } @@ -339,7 +339,7 @@ impl<'a> TableConfig<'a> { /// Return the column mapping mode according to delta.columnMapping.mode pub fn column_mapping_mode(&self) -> ColumnMappingMode { self.0 - .get(DeltaConfigKey::ColumnMappingMode.as_ref()) + .get(TableProperty::ColumnMappingMode.as_ref()) .and_then(|o| o.as_ref().and_then(|v| v.parse().ok())) .unwrap_or_default() } @@ -362,7 +362,7 @@ impl<'a> TableConfig<'a> { /// This property takes precedence over [num_indexed_cols](Self::num_indexed_cols). pub fn stats_columns(&self) -> Option> { self.0 - .get(DeltaConfigKey::DataSkippingStatsColumns.as_ref()) + .get(TableProperty::DataSkippingStatsColumns.as_ref()) .and_then(|o| o.as_ref().map(|v| v.split(',').collect())) } } @@ -536,7 +536,7 @@ mod tests { // change to 2 day let mut md = dummy_metadata(); md.configuration.insert( - DeltaConfigKey::DeletedFileRetentionDuration + TableProperty::DeletedFileRetentionDuration .as_ref() .to_string(), Some("interval 2 day".to_string()), @@ -567,7 +567,7 @@ mod tests { // change to false let mut md = dummy_metadata(); md.configuration.insert( - DeltaConfigKey::EnableExpiredLogCleanup.as_ref().into(), + TableProperty::EnableExpiredLogCleanup.as_ref().into(), Some("false".to_string()), ); let config = TableConfig(&md.configuration); diff --git a/crates/core/src/writer/test_utils.rs b/crates/core/src/writer/test_utils.rs index a75f17e7cf..be0dfebb66 100644 --- a/crates/core/src/writer/test_utils.rs +++ b/crates/core/src/writer/test_utils.rs @@ -10,7 +10,7 @@ use arrow_select::take::take; use crate::kernel::{DataType as DeltaDataType, Metadata, PrimitiveType, StructField, StructType}; use crate::operations::create::CreateBuilder; use crate::operations::DeltaOps; -use crate::{DeltaConfigKey, DeltaTable, DeltaTableBuilder}; +use crate::{DeltaTable, DeltaTableBuilder, TableProperty}; pub type TestResult = Result<(), Box>; @@ -270,7 +270,7 @@ pub fn get_delta_schema_with_nested_struct() -> StructType { } pub async fn setup_table_with_configuration( - key: DeltaConfigKey, + key: TableProperty, value: Option>, ) -> DeltaTable { let table_schema = get_delta_schema(); diff --git a/crates/core/tests/checkpoint_writer.rs b/crates/core/tests/checkpoint_writer.rs index 696e379569..1be439f9e5 100644 --- a/crates/core/tests/checkpoint_writer.rs +++ b/crates/core/tests/checkpoint_writer.rs @@ -87,7 +87,7 @@ mod delete_expired_delta_log_in_checkpoint { use ::object_store::path::Path as ObjectStorePath; use chrono::Utc; - use deltalake_core::table::config::DeltaConfigKey; + use deltalake_core::table::config::TableProperty; use deltalake_core::*; use maplit::hashmap; @@ -96,8 +96,8 @@ mod delete_expired_delta_log_in_checkpoint { let mut table = fs_common::create_table( "../test/tests/data/checkpoints_with_expired_logs/expired", Some(hashmap! { - DeltaConfigKey::LogRetentionDuration.as_ref().into() => Some("interval 10 minute".to_string()), - DeltaConfigKey::EnableExpiredLogCleanup.as_ref().into() => Some("true".to_string()) + TableProperty::LogRetentionDuration.as_ref().into() => Some("interval 10 minute".to_string()), + TableProperty::EnableExpiredLogCleanup.as_ref().into() => Some("true".to_string()) }), ) .await; @@ -160,8 +160,8 @@ mod delete_expired_delta_log_in_checkpoint { let mut table = fs_common::create_table( "../test/tests/data/checkpoints_with_expired_logs/not_delete_expired", Some(hashmap! { - DeltaConfigKey::LogRetentionDuration.as_ref().into() => Some("interval 1 second".to_string()), - DeltaConfigKey::EnableExpiredLogCleanup.as_ref().into() => Some("false".to_string()) + TableProperty::LogRetentionDuration.as_ref().into() => Some("interval 1 second".to_string()), + TableProperty::EnableExpiredLogCleanup.as_ref().into() => Some("false".to_string()) }), ) .await; @@ -208,7 +208,7 @@ mod checkpoints_with_tombstones { use ::object_store::path::Path as ObjectStorePath; use chrono::Utc; use deltalake_core::kernel::*; - use deltalake_core::table::config::DeltaConfigKey; + use deltalake_core::table::config::TableProperty; use deltalake_core::*; use maplit::hashmap; use parquet::file::reader::{FileReader, SerializedFileReader}; @@ -235,7 +235,7 @@ mod checkpoints_with_tombstones { #[ignore] async fn test_expired_tombstones() { let mut table = fs_common::create_table("../test/tests/data/checkpoints_tombstones/expired", Some(hashmap! { - DeltaConfigKey::DeletedFileRetentionDuration.as_ref().into() => Some("interval 1 minute".to_string()) + TableProperty::DeletedFileRetentionDuration.as_ref().into() => Some("interval 1 minute".to_string()) })).await; let a1 = fs_common::add(3 * 60 * 1000); // 3 mins ago, diff --git a/crates/deltalake/Cargo.toml b/crates/deltalake/Cargo.toml index b411c4ccd5..4d4df632c5 100644 --- a/crates/deltalake/Cargo.toml +++ b/crates/deltalake/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake" -version = "0.19.1" +version = "0.20.0" authors.workspace = true keywords.workspace = true readme.workspace = true @@ -16,12 +16,12 @@ rust-version.workspace = true features = ["azure", "datafusion", "gcs", "hdfs", "json", "python", "s3", "unity-experimental"] [dependencies] -deltalake-core = { version = "0.19.1", path = "../core" } -deltalake-aws = { version = "0.2.0", path = "../aws", default-features = false, optional = true } -deltalake-azure = { version = "0.2.0", path = "../azure", optional = true } -deltalake-gcp = { version = "0.3.0", path = "../gcp", optional = true } -deltalake-hdfs = { version = "0.3.0", path = "../hdfs", optional = true } -deltalake-catalog-glue = { version = "0.3.0", path = "../catalog-glue", optional = true } +deltalake-core = { version = "0.20.0", path = "../core" } +deltalake-aws = { version = "0.3.0", path = "../aws", default-features = false, optional = true } +deltalake-azure = { version = "0.3.0", path = "../azure", optional = true } +deltalake-gcp = { version = "0.4.0", path = "../gcp", optional = true } +deltalake-hdfs = { version = "0.4.0", path = "../hdfs", optional = true } +deltalake-catalog-glue = { version = "0.4.0", path = "../catalog-glue", optional = true } [features] # All of these features are just reflected into the core crate until that diff --git a/crates/gcp/Cargo.toml b/crates/gcp/Cargo.toml index 623868a6aa..380aa84852 100644 --- a/crates/gcp/Cargo.toml +++ b/crates/gcp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-gcp" -version = "0.3.0" +version = "0.4.0" authors.workspace = true keywords.workspace = true readme.workspace = true @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.1", path = "../core" } +deltalake-core = { version = "0.20.0", path = "../core" } lazy_static = "1" # workspace depenndecies diff --git a/crates/hdfs/Cargo.toml b/crates/hdfs/Cargo.toml index bb9525c302..f601f55c6d 100644 --- a/crates/hdfs/Cargo.toml +++ b/crates/hdfs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-hdfs" -version = "0.3.0" +version = "0.4.0" authors.workspace = true keywords.workspace = true readme.workspace = true @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.1", path = "../core" } +deltalake-core = { version = "0.20.0", path = "../core" } hdfs-native-object-store = "0.11" # workspace dependecies diff --git a/crates/mount/Cargo.toml b/crates/mount/Cargo.toml index 2e8e42eccf..a770200b98 100644 --- a/crates/mount/Cargo.toml +++ b/crates/mount/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-mount" -version = "0.3.0" +version = "0.4.0" authors.workspace = true keywords.workspace = true readme.workspace = true @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.1", path = "../core", features = [ +deltalake-core = { version = "0.20.0", path = "../core", features = [ "datafusion", ] } lazy_static = "1" diff --git a/crates/test/Cargo.toml b/crates/test/Cargo.toml index 20fb238a5f..3638e6fefa 100644 --- a/crates/test/Cargo.toml +++ b/crates/test/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "deltalake-test" -version = "0.2.0" +version = "0.3.0" edition = "2021" publish = false [dependencies] bytes = { workspace = true } chrono = { workspace = true, default-features = false, features = ["clock"] } -deltalake-core = { version = ">=0.17.0, <0.20.0", path = "../core" } +deltalake-core = { version = "0.20.0", path = "../core" } dotenvy = "0" fs_extra = "1.3.0" futures = { version = "0.3" }