Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Low level create table #342

Merged
merged 13 commits into from
Aug 4, 2021
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions rust/src/action.rs
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,14 @@ pub struct Format {
options: Option<HashMap<String, String>>,
}

impl Format {
pub fn new(provider: String, options: Option<HashMap<String,String>>) -> Self {
Smurphy000 marked this conversation as resolved.
Show resolved Hide resolved
Self {
provider,
options
}
}
}
/// Action that describes the metadata of the table.
/// This is a top-level action in Delta log entries.
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
Expand Down
91 changes: 91 additions & 0 deletions rust/src/delta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,29 @@ pub struct DeltaTableMetaData {
pub configuration: HashMap<String, String>,
}

impl DeltaTableMetaData {
fn new(
name: Option<String>,
description: Option<String>,
schema: Schema,
partition: Vec<String>,
config: HashMap<String, String>
) -> Self {
//use inputs

Self {
id: Uuid::new_v4().to_string(), //create a new GUID
Smurphy000 marked this conversation as resolved.
Show resolved Hide resolved
name: name,
description: description,
format: action::Format::new("parquet".to_string(), None),
schema: schema,
partition_columns: partition,
created_time: Utc::now().timestamp(), //create a timestamp for current timestamp
Smurphy000 marked this conversation as resolved.
Show resolved Hide resolved
configuration: config
}
}
}

impl fmt::Display for DeltaTableMetaData {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
Expand Down Expand Up @@ -1034,6 +1057,30 @@ impl DeltaTable {
})
}

pub async fn create(
Smurphy000 marked this conversation as resolved.
Show resolved Hide resolved
&mut self,
metadata: DeltaTableMetaData,
protocol: action::Protocol
) -> Result<(), DeltaTableError> {
// need to check if table already exists so that protocal and metaData arent overwritten on second create

let meta = action::MetaData::try_from(metadata).unwrap();
Smurphy000 marked this conversation as resolved.
Show resolved Hide resolved

let mut transaction = self.create_transaction(None);

//add commit info action
transaction.add_actions(
vec![
Action::protocol(protocol),
Action::metaData(meta)
]
);
transaction.commit(None).await.unwrap();
Smurphy000 marked this conversation as resolved.
Show resolved Hide resolved

//possible need to refresh state to give an accurate DeltaTable reference back at the end of creation
Ok(())
}

/// Time travel Delta table to latest version that's created at or before provided `datetime`
/// argument.
///
Expand Down Expand Up @@ -1577,4 +1624,48 @@ mod tests {
assert!(parquet_filename.contains("col1=a/col2=b/part-00000-"));
}
}

#[tokio::test]
async fn test_create_delta_table() {

let test_schema = Schema::new(
"test".to_string(),
vec![
SchemaField::new(
"Id".to_string(),
SchemaDataType::primitive("integer".to_string()),
true,
HashMap::new()
),
SchemaField::new(
"Name".to_string(),
SchemaDataType::primitive("string".to_string()),
true,
HashMap::new()
)
]
);

let deltamd = DeltaTableMetaData::new(
None,
None,
test_schema,
vec![],
HashMap::new()
);
Smurphy000 marked this conversation as resolved.
Show resolved Hide resolved

let protocol = action::Protocol{
min_reader_version: 1,
min_writer_version: 2
};

let mut dt = DeltaTable::new(
"./test_create/",
Box::new(storage::file::FileStorageBackend::new("./"))
).unwrap();

let _new_delta_table = dt.create(deltamd, protocol).await.unwrap();

//clean up data created
}
}
22 changes: 22 additions & 0 deletions rust/src/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,21 @@ pub struct SchemaField {
}

impl SchemaField {

pub fn new(
name: String,
r#type: SchemaDataType,
nullable: bool,
metadata: HashMap<String, String>
) -> Self {
Self {
name,
r#type,
nullable,
metadata
}
}

/// The column name of the schema field.
pub fn get_name(&self) -> &str {
&self.name
Expand Down Expand Up @@ -158,4 +173,11 @@ impl Schema {
pub fn get_fields(&self) -> &Vec<SchemaField> {
&self.fields
}

pub fn new(r#type: String, fields: Vec<SchemaField>) -> Self {
Self {
r#type,
fields
}
}
}