-
Notifications
You must be signed in to change notification settings - Fork 147
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[FEAT]: hash expr #2398
[FEAT]: hash expr #2398
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
use common_error::{DaftError, DaftResult}; | ||
use daft_core::{ | ||
datatypes::{Field, UInt64Array}, | ||
schema::Schema, | ||
DataType, IntoSeries, Series, | ||
}; | ||
|
||
use crate::{ | ||
functions::{FunctionEvaluator, FunctionExpr}, | ||
Expr, ExprRef, | ||
}; | ||
|
||
pub(super) struct HashEvaluator {} | ||
|
||
impl FunctionEvaluator for HashEvaluator { | ||
fn fn_name(&self) -> &'static str { | ||
"hash" | ||
} | ||
|
||
fn to_field(&self, inputs: &[ExprRef], schema: &Schema, _: &FunctionExpr) -> DaftResult<Field> { | ||
match inputs { | ||
[input] | [input, _] => match input.to_field(schema) { | ||
Ok(field) => Ok(Field::new(field.name, DataType::UInt64)), | ||
e => e, | ||
}, | ||
_ => Err(DaftError::SchemaMismatch(format!( | ||
"Expected 2 input arg, got {}", | ||
inputs.len() | ||
))), | ||
} | ||
} | ||
|
||
fn evaluate(&self, inputs: &[Series], _: &FunctionExpr) -> DaftResult<Series> { | ||
match inputs { | ||
[input] => input.hash(None).map(|s| s.into_series()), | ||
[input, seed] => { | ||
if seed.len() != 1 { | ||
return Err(DaftError::ValueError("Expected 1 seed arg".to_string())); | ||
} | ||
|
||
let seed = seed.cast(&DataType::UInt64)?; | ||
// There's no way to natively extend the array, so we extract the element and repeat it. | ||
let seed = seed.u64().unwrap(); | ||
let seed = seed.get(0).unwrap(); | ||
let seed = | ||
UInt64Array::from_iter("seed", std::iter::repeat(Some(seed)).take(input.len())); | ||
|
||
input.hash(Some(&seed)).map(|s| s.into_series()) | ||
} | ||
_ => Err(DaftError::ValueError("Expected 2 input arg".to_string())), | ||
} | ||
} | ||
} | ||
|
||
pub fn hash(input: ExprRef, seed: Option<ExprRef>) -> ExprRef { | ||
let inputs = match seed { | ||
Some(seed) => vec![input, seed], | ||
None => vec![input], | ||
}; | ||
|
||
Expr::Function { | ||
func: FunctionExpr::Hash, | ||
inputs, | ||
} | ||
.into() | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -249,6 +249,24 @@ def test_repr_functions_sqrt() -> None: | |
assert repr_out == repr(copied) | ||
|
||
|
||
def test_repr_functions_hash() -> None: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It would also be good to run some of these hash tests that we run on the series on the expression as well! https://github.com/Eventual-Inc/Daft/blob/main/tests/series/test_hash.py There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I was thinking that too, but didn't see anywhere else the exprs were being tested like that. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We have tests like these here in You can create a new file like |
||
a = col("a") | ||
y = a.hash() | ||
repr_out = repr(y) | ||
assert repr_out == "hash(col(a))" | ||
copied = copy.deepcopy(y) | ||
assert repr_out == repr(copied) | ||
|
||
|
||
def test_repr_functions_hash_2() -> None: | ||
a = col("a") | ||
y = a.hash(lit(1)) | ||
repr_out = repr(y) | ||
assert repr_out == "hash(col(a), lit(1))" | ||
copied = copy.deepcopy(y) | ||
assert repr_out == repr(copied) | ||
|
||
|
||
def test_expr_structurally_equal() -> None: | ||
e1 = (col("a").max() == col("b").alias("moo") - 3).is_null() | ||
e2 = (col("a").max() == col("b").alias("moo") - 3).is_null() | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should also be able to use seed if the length is equal to input like in the case
x.hash(seed=y.hash())