Skip to content

Commit

Permalink
fix(deps): update dependency pyarrow to v13
Browse files Browse the repository at this point in the history
  • Loading branch information
renovate[bot] authored and cpcloud committed Sep 30, 2023
1 parent 6bdec79 commit 43dc1e1
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 37 deletions.
6 changes: 1 addition & 5 deletions ibis/backends/pandas/execution/temporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,7 @@ def execute_extract_microsecond_series(op, data, **kwargs):

@execute_node.register(ops.ExtractEpochSeconds, (datetime.datetime, pd.Series))
def execute_epoch_seconds(op, data, **kwargs):
# older versions of dask do not have a view method, so use astype
# instead
convert = getattr(data, "view", data.astype)
series = convert(np.int64)
return (series // 1_000_000_000).astype(np.int32)
return data.astype("datetime64[s]").astype("int64").astype("int32")


@execute_node.register(
Expand Down
6 changes: 3 additions & 3 deletions ibis/backends/tests/test_aggregation.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def test_aggregate(backend, alltypes, df, result_fn, expected_fn):
# (to match the output format of Ibis `aggregate`)
expected = pd.DataFrame({"tmp": [expected_fn(df)]})

backend.assert_frame_equal(result, expected)
backend.assert_frame_equal(result, expected, check_dtype=False)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -187,8 +187,8 @@ def test_aggregate_grouped(backend, alltypes, df, result_fn, expected_fn):
result2 = result2.sort_values(by=grouping_key_col).reset_index(drop=True)
expected = expected.sort_values(by=grouping_key_col).reset_index(drop=True)

backend.assert_frame_equal(result1, expected)
backend.assert_frame_equal(result2, expected)
backend.assert_frame_equal(result1, expected, check_dtype=False)
backend.assert_frame_equal(result2, expected, check_dtype=False)


@mark.notimpl(
Expand Down
20 changes: 10 additions & 10 deletions ibis/backends/tests/test_temporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ def test_timestamp_extract_epoch_seconds(backend, alltypes, df):
result = expr.execute()

expected = backend.default_series_rename(
(df.timestamp_col.view("int64") // 1_000_000_000).astype("int32")
df.timestamp_col.astype("datetime64[s]").astype("int64").astype("int32")
)
backend.assert_series_equal(result, expected)

Expand Down Expand Up @@ -547,7 +547,7 @@ def test_date_truncate(backend, alltypes, df, unit):
result = expr.execute()
expected = backend.default_series_rename(expected)

backend.assert_series_equal(result, expected)
backend.assert_series_equal(result, expected.astype(result.dtype))


@pytest.mark.parametrize(
Expand Down Expand Up @@ -732,7 +732,7 @@ def convert_to_offset(offset, displacement_type=displacement_type):
expected = df.timestamp_col + offset

expected = backend.default_series_rename(expected)
backend.assert_series_equal(result, expected.astype("datetime64[ns]"))
backend.assert_series_equal(result, expected.astype(result.dtype))


@pytest.mark.parametrize(
Expand Down Expand Up @@ -975,7 +975,7 @@ def test_temporal_binop(backend, con, alltypes, df, expr_fn, expected_fn):
result = con.execute(expr)
expected = backend.default_series_rename(expected)

backend.assert_series_equal(result, expected)
backend.assert_series_equal(result, expected.astype(result.dtype))


plus = lambda t, td: t.timestamp_col + pd.Timedelta(td)
Expand Down Expand Up @@ -1152,7 +1152,7 @@ def test_temporal_binop_pandas_timedelta(
result = con.execute(expr)
expected = backend.default_series_rename(expected)

backend.assert_series_equal(result, expected)
backend.assert_series_equal(result, expected.astype(result.dtype))


@pytest.mark.parametrize("func_name", ["gt", "ge", "lt", "le", "eq", "ne"])
Expand Down Expand Up @@ -1296,7 +1296,7 @@ def test_interval_add_cast_scalar(backend, alltypes):
expr = (timestamp_date + delta).name("result")
result = expr.execute()
expected = timestamp_date.name("result").execute() + pd.Timedelta(10, unit="D")
backend.assert_series_equal(result, expected)
backend.assert_series_equal(result, expected.astype(result.dtype))


@pytest.mark.never(
Expand All @@ -1323,7 +1323,7 @@ def test_interval_add_cast_column(backend, alltypes, df):
.add(df.bigint_col.astype("timedelta64[D]"))
.rename("tmp")
)
backend.assert_series_equal(result, expected)
backend.assert_series_equal(result, expected.astype(result.dtype))


@pytest.mark.parametrize(
Expand Down Expand Up @@ -2038,7 +2038,7 @@ def test_date_column_from_ymd(con, alltypes, df):
tbl = alltypes[expr.name("timestamp_col")]
result = con.execute(tbl)

golden = df.timestamp_col.dt.date.astype("datetime64[ns]")
golden = df.timestamp_col.dt.date.astype(result.timestamp_col.dtype)
tm.assert_series_equal(golden, result.timestamp_col)


Expand Down Expand Up @@ -2068,7 +2068,7 @@ def test_timestamp_column_from_ymdhms(con, alltypes, df):
tbl = alltypes[expr.name("timestamp_col")]
result = con.execute(tbl)

golden = df.timestamp_col.dt.floor("s").astype("datetime64[ns]")
golden = df.timestamp_col.dt.floor("s").astype(result.timestamp_col.dtype)
tm.assert_series_equal(golden, result.timestamp_col)


Expand Down Expand Up @@ -2169,7 +2169,7 @@ def test_integer_cast_to_timestamp_column(backend, alltypes, df):
expr = alltypes.int_col.cast("timestamp")
expected = pd.to_datetime(df.int_col, unit="s").rename(expr.get_name())
result = expr.execute()
backend.assert_series_equal(result, expected)
backend.assert_series_equal(result, expected.astype(result.dtype))


@pytest.mark.notimpl(
Expand Down
32 changes: 14 additions & 18 deletions ibis/backends/tests/test_timecontext.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

import ibis
from ibis.backends.tests.test_vectorized_udf import calc_mean, create_demean_struct_udf
from ibis.config import option_context

pytestmark = pytest.mark.notimpl(
[
Expand Down Expand Up @@ -41,7 +40,8 @@ def context():


def filter_by_time_context(df, context):
return df[(df["timestamp_col"] >= context[0]) & (df["timestamp_col"] < context[1])]
begin, end = context
return df[(df.timestamp_col >= begin) & (df.timestamp_col < end)]


broken_pandas_grouped_rolling = pytest.mark.xfail(
Expand All @@ -51,14 +51,8 @@ def filter_by_time_context(df, context):
)


@pytest.fixture(scope="module")
def ctx_col():
with option_context("context_adjustment.time_col", "timestamp_col"):
yield


@pytest.mark.notimpl(["dask", "duckdb"])
@pytest.mark.xfail_version(pyspark=["pyspark<3.1"])
@pytest.mark.xfail_version(pyspark=["pyspark<3.1"], pandas=["pyarrow>=13", "pandas>=2"])
@pytest.mark.parametrize(
"window",
[
Expand All @@ -77,9 +71,10 @@ def ctx_col():
),
],
)
def test_context_adjustment_window_udf(alltypes, context, window, ctx_col):
"""This test case aims to test context adjustment of udfs in window
method."""
def test_context_adjustment_window_udf(alltypes, context, window, monkeypatch):
"""Test context adjustment of udfs in window methods."""
monkeypatch.setattr(ibis.options.context_adjustment, "time_col", "timestamp_col")

expr = alltypes.mutate(v1=calc_mean(alltypes[TARGET_COL]).over(window))
result = expr.execute(timecontext=context)

Expand All @@ -90,7 +85,10 @@ def test_context_adjustment_window_udf(alltypes, context, window, ctx_col):


@pytest.mark.notimpl(["dask", "duckdb"])
def test_context_adjustment_filter_before_window(alltypes, context, ctx_col):
@pytest.mark.xfail_version(pandas=["pyarrow>=13", "pandas>=2"])
def test_context_adjustment_filter_before_window(alltypes, context, monkeypatch):
monkeypatch.setattr(ibis.options.context_adjustment, "time_col", "timestamp_col")

window = ibis.trailing_window(ibis.interval(days=3), order_by=ORDER_BY_COL)

expr = alltypes[alltypes["bool_col"]]
Expand All @@ -106,11 +104,9 @@ def test_context_adjustment_filter_before_window(alltypes, context, ctx_col):


@pytest.mark.notimpl(["duckdb", "pyspark"])
def test_context_adjustment_multi_col_udf_non_grouped(
alltypes,
context,
ctx_col,
):
def test_context_adjustment_multi_col_udf_non_grouped(alltypes, context, monkeypatch):
monkeypatch.setattr(ibis.options.context_adjustment, "time_col", "timestamp_col")

w = ibis.window(preceding=None, following=None)

demean_struct_udf = create_demean_struct_udf(
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ numpy = ">=1,<2"
pandas = ">=1.2.5,<3"
parsy = ">=2,<3"
pins = { version = ">=0.8.2,<1", extras = ["gcs"] }
pyarrow = ">=2,<13"
pyarrow = ">=2,<14"
python-dateutil = ">=2.8.2,<3"
pytz = ">=2022.7"
rich = ">=12.4.4,<14"
Expand Down

0 comments on commit 43dc1e1

Please sign in to comment.