Skip to content

Commit

Permalink
feat: add microsecond method to TimestampValue and TimeValue
Browse files Browse the repository at this point in the history
  • Loading branch information
thejaredchapman authored and cpcloud committed Jun 22, 2023
1 parent 6dafe42 commit e9df2da
Show file tree
Hide file tree
Showing 18 changed files with 69 additions and 1 deletion.
3 changes: 3 additions & 0 deletions docs/api/expressions/timestamps.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,9 @@ All temporal operations are valid for both scalars and columns.
::: ibis.expr.types.temporal._TimeComponentMixin.hour
options:
heading_level: 4
::: ibis.expr.types.temporal._TimeComponentMixin.microsecond
options:
heading_level: 4
::: ibis.expr.types.temporal._TimeComponentMixin.millisecond
options:
heading_level: 4
Expand Down
1 change: 1 addition & 0 deletions ibis/backends/base/sql/registry/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,7 @@ def count_star(translator, op):
ops.ExtractHour: timestamp.extract_field('hour'),
ops.ExtractMinute: timestamp.extract_field('minute'),
ops.ExtractSecond: timestamp.extract_field('second'),
ops.ExtractMicrosecond: timestamp.extract_field('microsecond'),
ops.ExtractMillisecond: timestamp.extract_field('millisecond'),
ops.TimestampTruncate: timestamp.truncate,
ops.DateTruncate: timestamp.truncate,
Expand Down
1 change: 1 addition & 0 deletions ibis/backends/bigquery/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -679,6 +679,7 @@ def _interval_multiply(t, op):
ops.ExtractHour: _extract_field("hour"),
ops.ExtractMinute: _extract_field("minute"),
ops.ExtractSecond: _extract_field("second"),
ops.ExtractMicrosecond: _extract_field("microsecond"),
ops.ExtractMillisecond: _extract_field("millisecond"),
ops.Strftime: compiles_strftime,
ops.StringToTimestamp: compiles_string_to_timestamp,
Expand Down
6 changes: 6 additions & 0 deletions ibis/backends/clickhouse/compiler/values.py
Original file line number Diff line number Diff line change
Expand Up @@ -1097,6 +1097,12 @@ def _fmt(op, _name: str = _name, **kw):

del _fmt, _name, _op

@translate_val.register(ops.ExtractMicrosecond)
def _extract_microsecond(op, **kw):
arg = translate_val(op.arg, **kw)
dtype = serialize(op.output_dtype)
return f"CAST(substring(formatDateTime({arg}, '%f'), 1, 3) AS {dtype})"


@translate_val.register(ops.ExtractMillisecond)
def _extract_millisecond(op, **kw):
Expand Down
2 changes: 2 additions & 0 deletions ibis/backends/dask/execution/temporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
execute_day_of_week_index_series,
execute_day_of_week_name_series,
execute_epoch_seconds,
execute_extract_microsecond_series,
execute_extract_millisecond_series,
execute_extract_timestamp_field_series,
execute_interval_add_multiply_delta_series,
Expand All @@ -58,6 +59,7 @@
(((dd.Series,) + integer_types), execute_timestamp_from_unix)
],
ops.ExtractTemporalField: [((dd.Series,), execute_extract_timestamp_field_series)],
ops.ExtractMicrosecond: [((dd.Series,), execute_extract_microsecond_series)],
ops.ExtractMillisecond: [((dd.Series,), execute_extract_millisecond_series)],
ops.ExtractEpochSeconds: [((dd.Series,), execute_epoch_seconds)],
ops.IntervalFromInteger: [((dd.Series,), execute_interval_from_integer_series)],
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
extract(`i`, 'microsecond')
2 changes: 1 addition & 1 deletion ibis/backends/impala/tests/test_value_exprs.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ def test_negate(table, colname, snapshot):

@pytest.mark.parametrize(
"field",
['year', 'month', 'day', 'hour', 'minute', 'second', 'millisecond'],
['year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond', 'millisecond'],
)
def test_timestamp_extract_field(table, field, snapshot):
expr = getattr(table.i, field)()
Expand Down
3 changes: 3 additions & 0 deletions ibis/backends/mysql/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,9 @@ def compiles_mysql_trim(element, compiler, **kw):
ops.ExtractDayOfYear: unary(sa.func.dayofyear),
ops.ExtractEpochSeconds: unary(sa.func.UNIX_TIMESTAMP),
ops.ExtractWeekOfYear: unary(sa.func.weekofyear),
ops.ExtractMicrosecond: fixed_arity(
lambda arg: sa.func.floor(sa.extract('microsecond', arg)), 1
),
ops.ExtractMillisecond: fixed_arity(
lambda arg: sa.func.floor(sa.extract('microsecond', arg) / 1000), 1
),
Expand Down
1 change: 1 addition & 0 deletions ibis/backends/pandas/tests/execution/test_temporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
(lambda v: v.hour(), lambda vt: vt.hour),
(lambda v: v.minute(), lambda vt: vt.minute),
(lambda v: v.second(), lambda vt: vt.second),
(lambda v: v.microsecond(), lambda vt: int(vt.microsecond)),
(lambda v: v.millisecond(), lambda vt: int(vt.microsecond / 1e3)),
]
+ [
Expand Down
1 change: 1 addition & 0 deletions ibis/backends/polars/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -895,6 +895,7 @@ def unnest(op, **kw):
ops.ExtractHour: "hour",
ops.ExtractMinute: "minute",
ops.ExtractSecond: "second",
ops.ExtractMicrosecond: "microsecond",
ops.ExtractMillisecond: "millisecond",
}

Expand Down
5 changes: 5 additions & 0 deletions ibis/backends/pyspark/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1371,6 +1371,11 @@ def compile_extract_second(t, op, **kwargs):
return _extract_component_from_datetime(t, op, extract_fn=F.second, **kwargs)


@compiles(ops.ExtractMicrosecond)
def compile_extract_microsecond(t, op, **kwargs):
return _extract_component_from_datetime(t, op, extract_fn=F.microsecond, **kwargs)


@compiles(ops.ExtractMillisecond)
def compile_extract_millisecond(t, op, **kwargs):
raise com.UnsupportedOperationError(
Expand Down
6 changes: 6 additions & 0 deletions ibis/backends/snowflake/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,12 @@ def _map_get(t, op):
ops.RegexExtract: _regex_extract,
ops.RegexSearch: fixed_arity(sa.sql.operators.custom_op("REGEXP"), 2),
ops.RegexReplace: fixed_arity(sa.func.regexp_replace, 3),
ops.ExtractMicrosecond: fixed_arity(
lambda arg: sa.cast(
sa.extract("epoch_microsecond", arg) % 1000000, sa.SMALLINT
),
1,
),
ops.ExtractMillisecond: fixed_arity(
lambda arg: sa.cast(
sa.extract("epoch_millisecond", arg) % 1000, sa.SMALLINT
Expand Down
3 changes: 3 additions & 0 deletions ibis/backends/sqlite/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,9 @@ def _day_of_the_week_name(arg):
ops.ExtractHour: _strftime_int('%H'),
ops.ExtractMinute: _strftime_int('%M'),
ops.ExtractSecond: _strftime_int('%S'),
ops.ExtractMicrosecond: fixed_arity(
lambda arg: (sa.func.strftime('%f', arg)) % 1000, 1
),
ops.ExtractMillisecond: fixed_arity(
lambda arg: (sa.func.strftime('%f', arg) * 1000) % 1000, 1
),
Expand Down
22 changes: 22 additions & 0 deletions ibis/backends/tests/test_temporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,27 @@ def test_timestamp_extract_literal(con, func, expected):
assert con.execute(func(value).name("tmp")) == expected


@pytest.mark.notimpl(["datafusion", "oracle"], raises=com.OperationNotDefinedError)
@pytest.mark.broken(
["druid"],
raises=AttributeError,
reason="'StringColumn' object has no attribute 'microsecond'",
)
@pytest.mark.notyet(
["pyspark"],
raises=com.UnsupportedOperationError,
reason='PySpark backend does not support extracting microseconds.',
)

@pytest.mark.broken(["sqlite"], raises=AssertionError)
def test_timestamp_extract_microseconds(backend, alltypes, df):
expr = alltypes.timestamp_col.microsecond().name("microsecond")
result = expr.execute()
expected = backend.default_series_rename(
(df.timestamp_col.dt.microsecond).astype('int32')
).rename("microsecond")
backend.assert_series_equal(result, expected)

@pytest.mark.notimpl(["datafusion", "oracle"], raises=com.OperationNotDefinedError)
@pytest.mark.broken(
["druid"],
Expand All @@ -260,6 +281,7 @@ def test_timestamp_extract_literal(con, func, expected):
raises=com.UnsupportedOperationError,
reason='PySpark backend does not support extracting milliseconds.',
)

@pytest.mark.broken(["sqlite"], raises=AssertionError)
def test_timestamp_extract_milliseconds(backend, alltypes, df):
expr = alltypes.timestamp_col.millisecond().name("millisecond")
Expand Down
1 change: 1 addition & 0 deletions ibis/expr/decompile.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
ops.ExtractDayOfYear: "day_of_year",
ops.ExtractEpochSeconds: "epoch_seconds",
ops.ExtractHour: "hour",
ops.ExtractMicrosecond: "microsecond",
ops.ExtractMillisecond: "millisecond",
ops.ExtractMinute: "minute",
ops.ExtractMinute: "minute",
Expand Down
5 changes: 5 additions & 0 deletions ibis/expr/operations/temporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,11 @@ class ExtractSecond(ExtractTimeField):
pass


@public
class ExtractMicrosecond(ExtractTimeField):
pass


@public
class ExtractMillisecond(ExtractTimeField):
pass
Expand Down
4 changes: 4 additions & 0 deletions ibis/expr/types/temporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,10 @@ def second(self) -> ir.IntegerValue:
"""Extract the second component."""
return ops.ExtractSecond(self).to_expr()

def microsecond(self) -> ir.IntegerValue:
"""Extract the microsecond component."""
return ops.ExtractMicrosecond(self).to_expr()

def millisecond(self) -> ir.IntegerValue:
"""Extract the millisecond component."""
return ops.ExtractMillisecond(self).to_expr()
Expand Down
3 changes: 3 additions & 0 deletions ibis/tests/expr/test_timestamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def test_string_cast_to_timestamp(alltypes):
('hour', ops.ExtractHour, ir.IntegerColumn),
('minute', ops.ExtractMinute, ir.IntegerColumn),
('second', ops.ExtractSecond, ir.IntegerColumn),
('microsecond', ops.ExtractMicrosecond, ir.IntegerColumn),
('millisecond', ops.ExtractMillisecond, ir.IntegerColumn),
],
)
Expand Down Expand Up @@ -131,6 +132,7 @@ def test_timestamp_field_access_on_date(
('hour', ops.ExtractHour, ir.IntegerColumn),
('minute', ops.ExtractMinute, ir.IntegerColumn),
('second', ops.ExtractSecond, ir.IntegerColumn),
('microsecond', ops.ExtractMicrosecond, ir.IntegerColumn),
('millisecond', ops.ExtractMillisecond, ir.IntegerColumn),
],
)
Expand All @@ -148,6 +150,7 @@ def test_timestamp_field_access_on_date_failure(
('hour', ops.ExtractHour, ir.IntegerColumn),
('minute', ops.ExtractMinute, ir.IntegerColumn),
('second', ops.ExtractSecond, ir.IntegerColumn),
('microsecond', ops.ExtractMicrosecond, ir.IntegerColumn),
('millisecond', ops.ExtractMillisecond, ir.IntegerColumn),
],
)
Expand Down

0 comments on commit e9df2da

Please sign in to comment.