From bd129b0aa5dd7198814b69056de3a14cec97f97a Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Fri, 18 Aug 2023 11:10:48 -0500 Subject: [PATCH 01/18] pending changes --- python/pysrc/kaskada/_timestream.py | 6 ++++- python/pytests/aggregation/coalesce_test.py | 27 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 python/pytests/aggregation/coalesce_test.py diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index a9104ecf2..27001978c 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -874,7 +874,11 @@ def lookup(self, key: Union[Timestream, Literal]) -> Timestream: Timestream containing the lookup join between the `key` and `self`. """ return Timestream._call("lookup", key, self) - + + def coalesce(self, values: List[Timestream]) -> Timestream: + return Timestream._call("coalesce", self, values) + + def shift_to(self, time: Union[Timestream, datetime]) -> Timestream: """ Create a Timestream shifting each point forward to `time`. diff --git a/python/pytests/aggregation/coalesce_test.py b/python/pytests/aggregation/coalesce_test.py new file mode 100644 index 000000000..7812a2345 --- /dev/null +++ b/python/pytests/aggregation/coalesce_test.py @@ -0,0 +1,27 @@ +import kaskada as kd + +import pytest + + +@pytest.fixture(scope="module") +def source() -> kd.sources.CsvString: + content = "\n".join( + [ + "time,key,m,n", + "1996-12-19T16:39:57,A,5,10", + "1996-12-19T16:39:58,B,24,3", + "1996-12-19T16:39:59,A,17,6", + "1996-12-19T16:40:00,A,,9", + "1996-12-19T16:40:01,A,12,", + "1996-12-19T16:40:02,A,,", + ] + ) + return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") + + +def test_coalesce_unwindowed(source, golden) -> None: + m = source.col("m") + n = source.col("n") + golden.jsonl( + kd.record({"m": m, "coalesced_val": m.coalesce(values = [n])}) + ) From 287628f3d931816eb7bae9d1a675faa02230fed6 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 13:48:42 -0500 Subject: [PATCH 02/18] coalese --- .../reference/timestream/aggregation.md | 1 + python/pysrc/kaskada/_timestream.py | 20 ++++++++++++++----- python/pytests/aggregation/coalesce_test.py | 14 ++++++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/python/docs/source/reference/timestream/aggregation.md b/python/docs/source/reference/timestream/aggregation.md index 3f5740fce..2288a1932 100644 --- a/python/docs/source/reference/timestream/aggregation.md +++ b/python/docs/source/reference/timestream/aggregation.md @@ -15,6 +15,7 @@ Windowed: .. autosummary:: :toctree: ../apidocs/ + Timestream.coalesce Timestream.collect Timestream.count Timestream.count_if diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 27001978c..e40e35a77 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -874,11 +874,21 @@ def lookup(self, key: Union[Timestream, Literal]) -> Timestream: Timestream containing the lookup join between the `key` and `self`. """ return Timestream._call("lookup", key, self) - - def coalesce(self, values: List[Timestream]) -> Timestream: - return Timestream._call("coalesce", self, values) - - + + def coalesce( + self, arg: Timestream | Literal, *args: Timestream | Literal + ) -> Timestream: + """ + Create a Timestream for returning the first non-null value or null if all values are null. + + Args: + arg (Timestream | Literal): The next value to be coalesced (required). + + Returns: + Timestream containing the first non-null value from that row. If all values are null, then returns null. + """ + return Timestream._call("coalesce", self, arg, *args) + def shift_to(self, time: Union[Timestream, datetime]) -> Timestream: """ Create a Timestream shifting each point forward to `time`. diff --git a/python/pytests/aggregation/coalesce_test.py b/python/pytests/aggregation/coalesce_test.py index 7812a2345..e605f92b3 100644 --- a/python/pytests/aggregation/coalesce_test.py +++ b/python/pytests/aggregation/coalesce_test.py @@ -20,8 +20,20 @@ def source() -> kd.sources.CsvString: def test_coalesce_unwindowed(source, golden) -> None: + m = source.col("m") + n = source.col("n") + golden.jsonl(kd.record({"m": m, "n": n, "coalesced_val": m.coalesce(n)})) + + +def test_coalesce_since_true(source, golden) -> None: m = source.col("m") n = source.col("n") golden.jsonl( - kd.record({"m": m, "coalesced_val": m.coalesce(values = [n])}) + kd.record( + { + "m": m, + "n": n, + "coalesced_val": m.coalesce(n, window=kd.windows.Since(True)), + } + ) ) From 2d3479920e20bf32a5fe06676dd83e369cf184fa Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 13:55:25 -0500 Subject: [PATCH 03/18] removed windowed --- python/pytests/aggregation/coalesce_test.py | 13 ------------- .../coalesce_test/test_coalesce_unwindowed.jsonl | 6 ++++++ 2 files changed, 6 insertions(+), 13 deletions(-) create mode 100644 python/pytests/golden/coalesce_test/test_coalesce_unwindowed.jsonl diff --git a/python/pytests/aggregation/coalesce_test.py b/python/pytests/aggregation/coalesce_test.py index e605f92b3..cab7e5758 100644 --- a/python/pytests/aggregation/coalesce_test.py +++ b/python/pytests/aggregation/coalesce_test.py @@ -24,16 +24,3 @@ def test_coalesce_unwindowed(source, golden) -> None: n = source.col("n") golden.jsonl(kd.record({"m": m, "n": n, "coalesced_val": m.coalesce(n)})) - -def test_coalesce_since_true(source, golden) -> None: - m = source.col("m") - n = source.col("n") - golden.jsonl( - kd.record( - { - "m": m, - "n": n, - "coalesced_val": m.coalesce(n, window=kd.windows.Since(True)), - } - ) - ) diff --git a/python/pytests/golden/coalesce_test/test_coalesce_unwindowed.jsonl b/python/pytests/golden/coalesce_test/test_coalesce_unwindowed.jsonl new file mode 100644 index 000000000..5e2261f1c --- /dev/null +++ b/python/pytests/golden/coalesce_test/test_coalesce_unwindowed.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0,"coalesced_val":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0,"coalesced_val":24.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0,"coalesced_val":17.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0,"coalesced_val":9.0} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null,"coalesced_val":12.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null,"coalesced_val":null} From 8294b957757f8a13ca997045beb4b2f8fd19de88 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 14:14:15 -0500 Subject: [PATCH 04/18] added ceil --- python/pysrc/kaskada/_timestream.py | 10 ++++++++ python/pytests/aggregation/ceil_test.py | 24 +++++++++++++++++++ python/pytests/aggregation/coalesce_test.py | 1 - .../ceil_test/test_ceil_unwindowed.jsonl | 6 +++++ .../golden/time_test/test_time_of_point.jsonl | 8 +++---- 5 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 python/pytests/aggregation/ceil_test.py create mode 100644 python/pytests/golden/ceil_test/test_ceil_unwindowed.jsonl diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 064fb03fb..f7c562585 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -239,6 +239,16 @@ def __radd__(self, lhs: Union[Timestream, Literal]) -> Timestream: lhs = Timestream._literal(lhs, self._ffi_expr.session()) return lhs.add(self) + def ceil(self) -> Timestream: + """ + Create a Timestream for the number rounded up to the next largest integer. + + Returns: + Timestream + The Timestream resulting from the numeric ceiling of this. + """ + return Timestream._call("ceil", self) + def sub(self, rhs: Union[Timestream, Literal]) -> Timestream: """ Create a Timestream substracting `rhs` from this. diff --git a/python/pytests/aggregation/ceil_test.py b/python/pytests/aggregation/ceil_test.py new file mode 100644 index 000000000..af895cc13 --- /dev/null +++ b/python/pytests/aggregation/ceil_test.py @@ -0,0 +1,24 @@ +import kaskada as kd + +import pytest + + +@pytest.fixture(scope="module") +def source() -> kd.sources.CsvString: + content = "\n".join( + [ + "time,key,m", + "1996-12-19T16:39:57,A,5", + "1996-12-19T16:39:58,B,100.0001", + "1996-12-19T16:39:59,A,2.50", + "1996-12-19T16:40:00,A,", + "1996-12-19T16:40:01,A,0.99", + "1996-12-19T16:40:02,A,1.01", + ] + ) + return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") + + +def test_ceil_unwindowed(source, golden) -> None: + m = source.col("m") + golden.jsonl(kd.record({"m": m, "ceil_m": m.ceil()})) diff --git a/python/pytests/aggregation/coalesce_test.py b/python/pytests/aggregation/coalesce_test.py index cab7e5758..a1e38f523 100644 --- a/python/pytests/aggregation/coalesce_test.py +++ b/python/pytests/aggregation/coalesce_test.py @@ -23,4 +23,3 @@ def test_coalesce_unwindowed(source, golden) -> None: m = source.col("m") n = source.col("n") golden.jsonl(kd.record({"m": m, "n": n, "coalesced_val": m.coalesce(n)})) - diff --git a/python/pytests/golden/ceil_test/test_ceil_unwindowed.jsonl b/python/pytests/golden/ceil_test/test_ceil_unwindowed.jsonl new file mode 100644 index 000000000..fd2c34073 --- /dev/null +++ b/python/pytests/golden/ceil_test/test_ceil_unwindowed.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"ceil_m":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":100.0001,"ceil_m":101.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2.5,"ceil_m":3.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"ceil_m":null} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":0.99,"ceil_m":1.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":1.01,"ceil_m":2.0} diff --git a/python/pytests/golden/time_test/test_time_of_point.jsonl b/python/pytests/golden/time_test/test_time_of_point.jsonl index ae5bc1a7e..bd0b2ada0 100644 --- a/python/pytests/golden/time_test/test_time_of_point.jsonl +++ b/python/pytests/golden/time_test/test_time_of_point.jsonl @@ -1,4 +1,4 @@ -{"_time":"1996-12-19T16:39:57.000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"time_of_m":"1996-12-19T16:39:57.000","n":10,"time_of_n":"1996-12-19T16:39:57.000"} -{"_time":"1996-12-19T16:39:58.000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"time_of_m":"1996-12-19T16:39:58.000","n":3,"time_of_n":"1996-12-19T16:39:58.000"} -{"_time":"1996-12-19T16:39:59.000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"time_of_m":"1996-12-19T16:39:59.000","n":6,"time_of_n":"1996-12-19T16:39:59.000"} -{"_time":"1997-01-18T16:40:00.000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"time_of_m":"1997-01-18T16:40:00.000","n":9,"time_of_n":"1997-01-18T16:40:00.000"} +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"time_of_m":"1996-12-19T16:39:57.000000000","n":10,"time_of_n":"1996-12-19T16:39:57.000000000"} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"time_of_m":"1996-12-19T16:39:58.000000000","n":3,"time_of_n":"1996-12-19T16:39:58.000000000"} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"time_of_m":"1996-12-19T16:39:59.000000000","n":6,"time_of_n":"1996-12-19T16:39:59.000000000"} +{"_time":"1997-01-18T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"time_of_m":"1997-01-18T16:40:00.000000000","n":9,"time_of_n":"1997-01-18T16:40:00.000000000"} From 09ced22730754402908c92a272cd823b8f96f1f7 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 14:20:46 -0500 Subject: [PATCH 05/18] added clamp --- python/pysrc/kaskada/_timestream.py | 17 +++++++++++++ python/pytests/{aggregation => }/ceil_test.py | 0 python/pytests/clamp_test.py | 24 +++++++++++++++++++ .../{aggregation => }/coalesce_test.py | 0 .../golden/clamp_test/test_clamp.jsonl | 6 +++++ 5 files changed, 47 insertions(+) rename python/pytests/{aggregation => }/ceil_test.py (100%) create mode 100644 python/pytests/clamp_test.py rename python/pytests/{aggregation => }/coalesce_test.py (100%) create mode 100644 python/pytests/golden/clamp_test/test_clamp.jsonl diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index f7c562585..c08fd476b 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -249,6 +249,23 @@ def ceil(self) -> Timestream: """ return Timestream._call("ceil", self) + def clamp(self, min: Literal, max: Literal) -> Timestream: + """ + Create a Timestream clampped between the bounds of min and max. + + Parameters + ---------- + min : Literal + The literal value to set as the lower bound + max : Literal + The literal value to set as the upper bound + + Returns: + Timestream + The Timestream resulting from the clamped bounds between min and max. + """ + return Timestream._call("clamp", self, min, max) + def sub(self, rhs: Union[Timestream, Literal]) -> Timestream: """ Create a Timestream substracting `rhs` from this. diff --git a/python/pytests/aggregation/ceil_test.py b/python/pytests/ceil_test.py similarity index 100% rename from python/pytests/aggregation/ceil_test.py rename to python/pytests/ceil_test.py diff --git a/python/pytests/clamp_test.py b/python/pytests/clamp_test.py new file mode 100644 index 000000000..acaa0ce56 --- /dev/null +++ b/python/pytests/clamp_test.py @@ -0,0 +1,24 @@ +import kaskada as kd + +import pytest + + +@pytest.fixture(scope="module") +def source() -> kd.sources.CsvString: + content = "\n".join( + [ + "time,key,m", + "1996-12-19T16:39:57,A,5", + "1996-12-19T16:39:58,B,100.0001", + "1996-12-19T16:39:59,A,2.50", + "1996-12-19T16:40:00,A,", + "1996-12-19T16:40:01,A,0.99", + "1996-12-19T16:40:02,A,1.01", + ] + ) + return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") + + +def test_clamp(source, golden) -> None: + m = source.col("m") + golden.jsonl(kd.record({"m": m, "clamped_m": m.clamp(min=5, max=100)})) diff --git a/python/pytests/aggregation/coalesce_test.py b/python/pytests/coalesce_test.py similarity index 100% rename from python/pytests/aggregation/coalesce_test.py rename to python/pytests/coalesce_test.py diff --git a/python/pytests/golden/clamp_test/test_clamp.jsonl b/python/pytests/golden/clamp_test/test_clamp.jsonl new file mode 100644 index 000000000..8ef0c4bc8 --- /dev/null +++ b/python/pytests/golden/clamp_test/test_clamp.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"clamped_m":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":100.0001,"clamped_m":100.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2.5,"clamped_m":5.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"clamped_m":null} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":0.99,"clamped_m":5.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":1.01,"clamped_m":5.0} From ba19d98d9117e2ffe61a6017e63ac7998015cebf Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 14:31:44 -0500 Subject: [PATCH 06/18] added clamp tests --- python/pysrc/kaskada/_timestream.py | 4 +++- python/pytests/clamp_test.py | 12 +++++++++++- .../pytests/golden/clamp_test/test_clamp_max.jsonl | 6 ++++++ .../pytests/golden/clamp_test/test_clamp_min.jsonl | 6 ++++++ .../golden/clamp_test/test_clamp_min_max.jsonl | 6 ++++++ 5 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 python/pytests/golden/clamp_test/test_clamp_max.jsonl create mode 100644 python/pytests/golden/clamp_test/test_clamp_min.jsonl create mode 100644 python/pytests/golden/clamp_test/test_clamp_min_max.jsonl diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index c08fd476b..a020ae817 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -249,7 +249,9 @@ def ceil(self) -> Timestream: """ return Timestream._call("ceil", self) - def clamp(self, min: Literal, max: Literal) -> Timestream: + def clamp( + self, min: Union[Timestream, None] = None, max: Union[Timestream, None] = None + ) -> Timestream: """ Create a Timestream clampped between the bounds of min and max. diff --git a/python/pytests/clamp_test.py b/python/pytests/clamp_test.py index acaa0ce56..c163feed9 100644 --- a/python/pytests/clamp_test.py +++ b/python/pytests/clamp_test.py @@ -19,6 +19,16 @@ def source() -> kd.sources.CsvString: return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") -def test_clamp(source, golden) -> None: +def test_clamp_min_max(source, golden) -> None: m = source.col("m") golden.jsonl(kd.record({"m": m, "clamped_m": m.clamp(min=5, max=100)})) + + +def test_clamp_min(source, golden) -> None: + m = source.col("m") + golden.jsonl(kd.record({"m": m, "clamped_min": m.clamp(min=5)})) + + +def test_clamp_max(source, golden) -> None: + m = source.col("m") + golden.jsonl(kd.record({"m": m, "clamped_min": m.clamp(max=100)})) diff --git a/python/pytests/golden/clamp_test/test_clamp_max.jsonl b/python/pytests/golden/clamp_test/test_clamp_max.jsonl new file mode 100644 index 000000000..efc822574 --- /dev/null +++ b/python/pytests/golden/clamp_test/test_clamp_max.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"clamped_min":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":100.0001,"clamped_min":100.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2.5,"clamped_min":2.5} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"clamped_min":null} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":0.99,"clamped_min":0.99} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":1.01,"clamped_min":1.01} diff --git a/python/pytests/golden/clamp_test/test_clamp_min.jsonl b/python/pytests/golden/clamp_test/test_clamp_min.jsonl new file mode 100644 index 000000000..d0220a092 --- /dev/null +++ b/python/pytests/golden/clamp_test/test_clamp_min.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"clamped_min":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":100.0001,"clamped_min":100.0001} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2.5,"clamped_min":5.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"clamped_min":null} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":0.99,"clamped_min":5.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":1.01,"clamped_min":5.0} diff --git a/python/pytests/golden/clamp_test/test_clamp_min_max.jsonl b/python/pytests/golden/clamp_test/test_clamp_min_max.jsonl new file mode 100644 index 000000000..8ef0c4bc8 --- /dev/null +++ b/python/pytests/golden/clamp_test/test_clamp_min_max.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"clamped_m":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":100.0001,"clamped_m":100.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2.5,"clamped_m":5.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"clamped_m":null} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":0.99,"clamped_m":5.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":1.01,"clamped_m":5.0} From 007867dfa475a4c586fb40e4a0fa6919d8e6fa83 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 14:48:03 -0500 Subject: [PATCH 07/18] added exp --- python/pysrc/kaskada/_timestream.py | 11 +++++++++ python/pytests/exp_test.py | 24 +++++++++++++++++++ python/pytests/golden/exp_test/test_exp.jsonl | 6 +++++ 3 files changed, 41 insertions(+) create mode 100644 python/pytests/exp_test.py create mode 100644 python/pytests/golden/exp_test/test_exp.jsonl diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index a020ae817..23f93be6b 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -298,6 +298,17 @@ def __rsub__(self, lhs: Union[Timestream, Literal]) -> Timestream: lhs = Timestream._literal(lhs, self._ffi_expr.session()) return lhs.sub(self) + def exp(self) -> Timestream: + """ + Create a Timestreamp raising e to the power of this. + + Returns + ------- + Timestream + The Timestream resulting from `e^power`. + """ + return Timestream._call("exp", self) + def mul(self, rhs: Union[Timestream, Literal]) -> Timestream: """ Create a Timestream multiplying this and `rhs`. diff --git a/python/pytests/exp_test.py b/python/pytests/exp_test.py new file mode 100644 index 000000000..ab5d37bec --- /dev/null +++ b/python/pytests/exp_test.py @@ -0,0 +1,24 @@ +import kaskada as kd + +import pytest + + +@pytest.fixture(scope="module") +def source() -> kd.sources.CsvString: + content = "\n".join( + [ + "time,key,m", + "1996-12-19T16:39:57,A,1", + "1996-12-19T16:39:58,B,1", + "1996-12-19T16:39:59,A,2", + "1996-12-19T16:40:00,A,3", + "1996-12-19T16:40:01,A,4", + "1996-12-19T16:40:02,A,5", + ] + ) + return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") + + +def test_exp(source, golden) -> None: + m = source.col("m") + golden.jsonl(kd.record({"m": m, "exp_m": m.exp()})) diff --git a/python/pytests/golden/exp_test/test_exp.jsonl b/python/pytests/golden/exp_test/test_exp.jsonl new file mode 100644 index 000000000..bb951954a --- /dev/null +++ b/python/pytests/golden/exp_test/test_exp.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":1,"exp_m":2.7182818285} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":1,"exp_m":2.7182818285} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2,"exp_m":7.3890560989} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":3,"exp_m":20.0855369232} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":4,"exp_m":54.5981500331} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":5,"exp_m":148.4131591026} From aed9f1a89252b0ebdf8c7f8d5617aa2e184aa763 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 14:52:46 -0500 Subject: [PATCH 08/18] docs --- python/pysrc/kaskada/_timestream.py | 43 ++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 23f93be6b..e352b56b8 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -243,28 +243,30 @@ def ceil(self) -> Timestream: """ Create a Timestream for the number rounded up to the next largest integer. - Returns: - Timestream - The Timestream resulting from the numeric ceiling of this. + Returns + ------- + Timestream + The Timestream resulting from the numeric ceiling of this. """ return Timestream._call("ceil", self) def clamp( - self, min: Union[Timestream, None] = None, max: Union[Timestream, None] = None + self, min: Union[Literal, None] = None, max: Union[Literal, None] = None ) -> Timestream: """ Create a Timestream clampped between the bounds of min and max. Parameters ---------- - min : Literal + min : Union[Literal, None] The literal value to set as the lower bound - max : Literal + max : Union[Literal, None] The literal value to set as the upper bound - Returns: - Timestream - The Timestream resulting from the clamped bounds between min and max. + Returns + ------- + Timestream + The Timestream resulting from the clamped bounds between min and max. """ return Timestream._call("clamp", self, min, max) @@ -309,6 +311,17 @@ def exp(self) -> Timestream: """ return Timestream._call("exp", self) + def floor(self) -> Timestream: + """ + Create a Timestream for the number rounded down to the previous largest integer. + + Returns + ------- + Timestream + The Timestream resulting from the numeric ceiling of this. + """ + return Timestream._call("floor", self) + def mul(self, rhs: Union[Timestream, Literal]) -> Timestream: """ Create a Timestream multiplying this and `rhs`. @@ -916,15 +929,19 @@ def lookup(self, key: Union[Timestream, Literal]) -> Timestream: return Timestream._call("lookup", key, self) def coalesce( - self, arg: Timestream | Literal, *args: Timestream | Literal + self, arg: Union[Timestream, Literal], *args: Union[Timestream, Literal] ) -> Timestream: """ Create a Timestream for returning the first non-null value or null if all values are null. - Args: - arg (Timestream | Literal): The next value to be coalesced (required). + Parameters + ---------- + arg : Union[Timestream, Literal] + The next value to be coalesced (required). - Returns: + Returns + ------- + Timestream Timestream containing the first non-null value from that row. If all values are null, then returns null. """ return Timestream._call("coalesce", self, arg, *args) From 29698c8af4fd5f42c243a109f961d42228a37a10 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 14:54:07 -0500 Subject: [PATCH 09/18] added floor tests --- python/pysrc/kaskada/_timestream.py | 2 +- python/pytests/floor_test.py | 24 +++++++++++++++++++ .../golden/floor_test/test_floor.jsonl | 6 +++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 python/pytests/floor_test.py create mode 100644 python/pytests/golden/floor_test/test_floor.jsonl diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index e352b56b8..181dcfa88 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -318,7 +318,7 @@ def floor(self) -> Timestream: Returns ------- Timestream - The Timestream resulting from the numeric ceiling of this. + The Timestream resulting from the numeric floor of this. """ return Timestream._call("floor", self) diff --git a/python/pytests/floor_test.py b/python/pytests/floor_test.py new file mode 100644 index 000000000..8e50cfa05 --- /dev/null +++ b/python/pytests/floor_test.py @@ -0,0 +1,24 @@ +import kaskada as kd + +import pytest + + +@pytest.fixture(scope="module") +def source() -> kd.sources.CsvString: + content = "\n".join( + [ + "time,key,m", + "1996-12-19T16:39:57,A,5", + "1996-12-19T16:39:58,B,100.0001", + "1996-12-19T16:39:59,A,2.50", + "1996-12-19T16:40:00,A,", + "1996-12-19T16:40:01,A,0.99", + "1996-12-19T16:40:02,A,1.01", + ] + ) + return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") + + +def test_floor(source, golden) -> None: + m = source.col("m") + golden.jsonl(kd.record({"m": m, "floor_m": m.floor()})) diff --git a/python/pytests/golden/floor_test/test_floor.jsonl b/python/pytests/golden/floor_test/test_floor.jsonl new file mode 100644 index 000000000..a794c1f5f --- /dev/null +++ b/python/pytests/golden/floor_test/test_floor.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"floor_m":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":100.0001,"floor_m":100.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2.5,"floor_m":2.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"floor_m":null} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":0.99,"floor_m":0.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":1.01,"floor_m":1.0} From 4d8f0f5558dd1649f881620b485e3a05b0688cb3 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 14:59:29 -0500 Subject: [PATCH 10/18] added powf --- python/pysrc/kaskada/_timestream.py | 14 +++++++++++ .../powf_test/test_powf_unwindowed.jsonl | 6 +++++ python/pytests/powf_test.py | 25 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 python/pytests/golden/powf_test/test_powf_unwindowed.jsonl create mode 100644 python/pytests/powf_test.py diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 181dcfa88..52458c6f9 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -352,6 +352,20 @@ def __rmul__(self, lhs: Union[Timestream, Literal]) -> Timestream: lhs = Timestream._literal(lhs, self._ffi_expr.session()) return lhs.mul(self) + def powf(self, power: Union[Timestream, Literal]) -> Timestream: + """ + + Parameters + ---------- + power : Union[Timestream, Literal] + The Timestream or literal value to raise this by. + + Returns + ------- + Timestream: The Timestream resulting from `self ^ power`. + """ + return Timestream._call("powf", self, power) + def div(self, divisor: Union[Timestream, Literal]) -> Timestream: """ Create a Timestream by dividing this and `divisor`. diff --git a/python/pytests/golden/powf_test/test_powf_unwindowed.jsonl b/python/pytests/golden/powf_test/test_powf_unwindowed.jsonl new file mode 100644 index 000000000..2427bc809 --- /dev/null +++ b/python/pytests/golden/powf_test/test_powf_unwindowed.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":1.0,"powf":1.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":2.0,"powf":8.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":3.0,"powf":81.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":4.0,"powf":1024.0} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"powf":null} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"powf":null} diff --git a/python/pytests/powf_test.py b/python/pytests/powf_test.py new file mode 100644 index 000000000..2feff57b9 --- /dev/null +++ b/python/pytests/powf_test.py @@ -0,0 +1,25 @@ +import kaskada as kd + +import pytest + + +@pytest.fixture(scope="module") +def source() -> kd.sources.CsvString: + content = "\n".join( + [ + "time,key,m,n", + "1996-12-19T16:39:57,A,1,2", + "1996-12-19T16:39:58,B,2,3", + "1996-12-19T16:39:59,A,3,4", + "1996-12-19T16:40:00,A,4,5", + "1996-12-19T16:40:01,A,5,", + "1996-12-19T16:40:02,A,,6", + ] + ) + return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") + + +def test_powf_unwindowed(source, golden) -> None: + m = source.col("m") + n = source.col("n") + golden.jsonl(kd.record({"m": m, "powf": m.powf(n)})) From 5844f1a2f8c298f2be4170482a4317a18a3552a7 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 15:02:46 -0500 Subject: [PATCH 11/18] added docs for arithmetic --- python/docs/source/reference/timestream/arithmetic.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/docs/source/reference/timestream/arithmetic.md b/python/docs/source/reference/timestream/arithmetic.md index 6a0d2109e..9dc5c1084 100644 --- a/python/docs/source/reference/timestream/arithmetic.md +++ b/python/docs/source/reference/timestream/arithmetic.md @@ -15,6 +15,11 @@ See the notes on the specific functions for more information. :toctree: ../apidocs/ Timestream.add + Timestream.ceil + Timestream.clamp + Timestream.exp + Timestream.flor + Timestream.powf Timestream.sub Timestream.mul Timestream.div From 7b0798ec03bceb272d9f0f77e5495ab7c047f574 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 15:07:19 -0500 Subject: [PATCH 12/18] linting --- python/pysrc/kaskada/_timestream.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 52458c6f9..8bdd14972 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -956,7 +956,8 @@ def coalesce( Returns ------- Timestream - Timestream containing the first non-null value from that row. If all values are null, then returns null. + Timestream containing the first non-null value from that row. + If all values are null, then returns null. """ return Timestream._call("coalesce", self, arg, *args) From 33b9b8bf173947343d8d58fa002d090e7ecbd1c4 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Mon, 21 Aug 2023 15:16:25 -0500 Subject: [PATCH 13/18] linting --- python/docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/docs/source/conf.py b/python/docs/source/conf.py index 5321bd7ea..28a265f77 100644 --- a/python/docs/source/conf.py +++ b/python/docs/source/conf.py @@ -69,7 +69,7 @@ "github_repo": "kaskada", "github_version": "main", "doc_path": "kaskada/docs/source", - "analytics":{ + "analytics": { "google_analytics_id": "G-HR9E2E6TG4", }, } From d44b0f5cd56c014e900f68949dd4402046fcacaa Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Tue, 22 Aug 2023 11:26:18 -0500 Subject: [PATCH 14/18] code review comments --- .../reference/timestream/aggregation.md | 1 - .../source/reference/timestream/arithmetic.md | 2 +- .../docs/source/reference/timestream/misc.md | 1 + python/pysrc/kaskada/_timestream.py | 18 +++--- python/pytests/ceil_test.py | 2 +- python/pytests/clamp_test.py | 58 +++++++++++++++++++ python/pytests/coalesce_test.py | 23 +++++--- .../pytests/golden/ceil_test/test_ceil.jsonl | 6 ++ .../clamp_test/test_clamp_banking_max.jsonl | 6 ++ .../clamp_test/test_clamp_banking_min.jsonl | 6 ++ .../test_clamp_banknig_min_max.jsonl | 6 ++ ...e_unwindowed.jsonl => test_coalesce.jsonl} | 0 .../coalesce_test/test_coalesce_three.jsonl | 6 ++ 13 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 python/pytests/golden/ceil_test/test_ceil.jsonl create mode 100644 python/pytests/golden/clamp_test/test_clamp_banking_max.jsonl create mode 100644 python/pytests/golden/clamp_test/test_clamp_banking_min.jsonl create mode 100644 python/pytests/golden/clamp_test/test_clamp_banknig_min_max.jsonl rename python/pytests/golden/coalesce_test/{test_coalesce_unwindowed.jsonl => test_coalesce.jsonl} (100%) create mode 100644 python/pytests/golden/coalesce_test/test_coalesce_three.jsonl diff --git a/python/docs/source/reference/timestream/aggregation.md b/python/docs/source/reference/timestream/aggregation.md index 2288a1932..3f5740fce 100644 --- a/python/docs/source/reference/timestream/aggregation.md +++ b/python/docs/source/reference/timestream/aggregation.md @@ -15,7 +15,6 @@ Windowed: .. autosummary:: :toctree: ../apidocs/ - Timestream.coalesce Timestream.collect Timestream.count Timestream.count_if diff --git a/python/docs/source/reference/timestream/arithmetic.md b/python/docs/source/reference/timestream/arithmetic.md index 9dc5c1084..669b7c810 100644 --- a/python/docs/source/reference/timestream/arithmetic.md +++ b/python/docs/source/reference/timestream/arithmetic.md @@ -18,7 +18,7 @@ See the notes on the specific functions for more information. Timestream.ceil Timestream.clamp Timestream.exp - Timestream.flor + Timestream.floor Timestream.powf Timestream.sub Timestream.mul diff --git a/python/docs/source/reference/timestream/misc.md b/python/docs/source/reference/timestream/misc.md index bfd18a25a..13880cb05 100644 --- a/python/docs/source/reference/timestream/misc.md +++ b/python/docs/source/reference/timestream/misc.md @@ -7,6 +7,7 @@ :toctree: ../apidocs/ Timestream.cast + Timestream.coalesce Timestream.data_type Timestream.else_ Timestream.filter diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 8bdd14972..679e79f2d 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -251,16 +251,18 @@ def ceil(self) -> Timestream: return Timestream._call("ceil", self) def clamp( - self, min: Union[Literal, None] = None, max: Union[Literal, None] = None + self, + min: Union[Timestream, Literal, None] = None, + max: Union[Timestream, Literal, None] = None, ) -> Timestream: """ - Create a Timestream clampped between the bounds of min and max. + Create a Timestream clamped between the bounds of min and max. Parameters ---------- - min : Union[Literal, None] + min : Union[Timestream, Literal, None] The literal value to set as the lower bound - max : Union[Literal, None] + max : Union[Timestream, Literal, None] The literal value to set as the upper bound Returns @@ -302,18 +304,18 @@ def __rsub__(self, lhs: Union[Timestream, Literal]) -> Timestream: def exp(self) -> Timestream: """ - Create a Timestreamp raising e to the power of this. + Create a Timestream raising `e` to the power of this. Returns ------- Timestream - The Timestream resulting from `e^power`. + The Timestream resulting from `e^this`. """ return Timestream._call("exp", self) def floor(self) -> Timestream: """ - Create a Timestream for the number rounded down to the previous largest integer. + Create a Timestream of the values rounded down to the nearest integer. Returns ------- @@ -952,6 +954,8 @@ def coalesce( ---------- arg : Union[Timestream, Literal] The next value to be coalesced (required). + args : Union[Timestream, Literal] + Additional values to be coalesced (optional). Returns ------- diff --git a/python/pytests/ceil_test.py b/python/pytests/ceil_test.py index af895cc13..9b2435ef2 100644 --- a/python/pytests/ceil_test.py +++ b/python/pytests/ceil_test.py @@ -19,6 +19,6 @@ def source() -> kd.sources.CsvString: return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") -def test_ceil_unwindowed(source, golden) -> None: +def test_ceil(source, golden) -> None: m = source.col("m") golden.jsonl(kd.record({"m": m, "ceil_m": m.ceil()})) diff --git a/python/pytests/clamp_test.py b/python/pytests/clamp_test.py index c163feed9..9e1307fdb 100644 --- a/python/pytests/clamp_test.py +++ b/python/pytests/clamp_test.py @@ -32,3 +32,61 @@ def test_clamp_min(source, golden) -> None: def test_clamp_max(source, golden) -> None: m = source.col("m") golden.jsonl(kd.record({"m": m, "clamped_min": m.clamp(max=100)})) + + +@pytest.fixture(scope="module") +def banking_source() -> kd.sources.CsvString: + content = "\n".join( + [ + "time,key,current_balance,min_balance,max_balance", + "1996-12-19T16:39:57,A,5.00,6.00,6.01", + "1996-12-19T16:39:58,B,6.00,7.00,7.01", + "1996-12-19T16:39:59,A,7.00,8.00,8.01", + "1996-12-19T16:40:00,A,8.00,9.00,9.01", + "1996-12-19T16:40:01,A,9.00,10.00,10.01", + "1996-12-19T16:40:02,A,10.00,11.00,11.01", + ] + ) + return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") + + +def test_clamp_banknig_min_max(banking_source, golden) -> None: + current_balance = banking_source.col("current_balance") + min_balance = banking_source.col("min_balance") + max_balance = banking_source.col("max_balance") + golden.jsonl( + kd.record( + { + "current_balance": current_balance, + "clamped_balance": current_balance.clamp( + min=min_balance, max=max_balance + ), + } + ) + ) + + +def test_clamp_banking_min(banking_source, golden) -> None: + current_balance = banking_source.col("current_balance") + min_balance = banking_source.col("min_balance") + golden.jsonl( + kd.record( + { + "current_balance": current_balance, + "clamped_balance": current_balance.clamp(min=min_balance), + } + ) + ) + + +def test_clamp_banking_max(banking_source, golden) -> None: + current_balance = banking_source.col("current_balance") + max_balance = banking_source.col("max_balance") + golden.jsonl( + kd.record( + { + "current_balance": current_balance, + "clamped_balance": current_balance.clamp(max=max_balance), + } + ) + ) diff --git a/python/pytests/coalesce_test.py b/python/pytests/coalesce_test.py index a1e38f523..8eef19d7c 100644 --- a/python/pytests/coalesce_test.py +++ b/python/pytests/coalesce_test.py @@ -7,19 +7,26 @@ def source() -> kd.sources.CsvString: content = "\n".join( [ - "time,key,m,n", - "1996-12-19T16:39:57,A,5,10", - "1996-12-19T16:39:58,B,24,3", - "1996-12-19T16:39:59,A,17,6", - "1996-12-19T16:40:00,A,,9", - "1996-12-19T16:40:01,A,12,", - "1996-12-19T16:40:02,A,,", + "time,key,m,n,o", + "1996-12-19T16:39:57,A,5,10,15", + "1996-12-19T16:39:58,B,24,3,15", + "1996-12-19T16:39:59,A,17,6,15", + "1996-12-19T16:40:00,A,,9,15", + "1996-12-19T16:40:01,A,12,,15", + "1996-12-19T16:40:02,A,,,15", ] ) return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") -def test_coalesce_unwindowed(source, golden) -> None: +def test_coalesce(source, golden) -> None: m = source.col("m") n = source.col("n") golden.jsonl(kd.record({"m": m, "n": n, "coalesced_val": m.coalesce(n)})) + + +def test_coalesce_three(source, golden) -> None: + m = source.col("m") + n = source.col("n") + o = source.col("o") + golden.jsonl(kd.record({"m": m, "n": n, "o": o, "coalesced_val": m.coalesce(n, o)})) diff --git a/python/pytests/golden/ceil_test/test_ceil.jsonl b/python/pytests/golden/ceil_test/test_ceil.jsonl new file mode 100644 index 000000000..fd2c34073 --- /dev/null +++ b/python/pytests/golden/ceil_test/test_ceil.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"ceil_m":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":100.0001,"ceil_m":101.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":2.5,"ceil_m":3.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"ceil_m":null} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":0.99,"ceil_m":1.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":1.01,"ceil_m":2.0} diff --git a/python/pytests/golden/clamp_test/test_clamp_banking_max.jsonl b/python/pytests/golden/clamp_test/test_clamp_banking_max.jsonl new file mode 100644 index 000000000..efd2696ff --- /dev/null +++ b/python/pytests/golden/clamp_test/test_clamp_banking_max.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","current_balance":5.0,"clamped_balance":5.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","current_balance":6.0,"clamped_balance":6.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","current_balance":7.0,"clamped_balance":7.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","current_balance":8.0,"clamped_balance":8.0} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","current_balance":9.0,"clamped_balance":9.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","current_balance":10.0,"clamped_balance":10.0} diff --git a/python/pytests/golden/clamp_test/test_clamp_banking_min.jsonl b/python/pytests/golden/clamp_test/test_clamp_banking_min.jsonl new file mode 100644 index 000000000..1a607b3c6 --- /dev/null +++ b/python/pytests/golden/clamp_test/test_clamp_banking_min.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","current_balance":5.0,"clamped_balance":6.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","current_balance":6.0,"clamped_balance":7.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","current_balance":7.0,"clamped_balance":8.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","current_balance":8.0,"clamped_balance":9.0} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","current_balance":9.0,"clamped_balance":10.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","current_balance":10.0,"clamped_balance":11.0} diff --git a/python/pytests/golden/clamp_test/test_clamp_banknig_min_max.jsonl b/python/pytests/golden/clamp_test/test_clamp_banknig_min_max.jsonl new file mode 100644 index 000000000..1a607b3c6 --- /dev/null +++ b/python/pytests/golden/clamp_test/test_clamp_banknig_min_max.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","current_balance":5.0,"clamped_balance":6.0} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","current_balance":6.0,"clamped_balance":7.0} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","current_balance":7.0,"clamped_balance":8.0} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","current_balance":8.0,"clamped_balance":9.0} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","current_balance":9.0,"clamped_balance":10.0} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","current_balance":10.0,"clamped_balance":11.0} diff --git a/python/pytests/golden/coalesce_test/test_coalesce_unwindowed.jsonl b/python/pytests/golden/coalesce_test/test_coalesce.jsonl similarity index 100% rename from python/pytests/golden/coalesce_test/test_coalesce_unwindowed.jsonl rename to python/pytests/golden/coalesce_test/test_coalesce.jsonl diff --git a/python/pytests/golden/coalesce_test/test_coalesce_three.jsonl b/python/pytests/golden/coalesce_test/test_coalesce_three.jsonl new file mode 100644 index 000000000..7f6163858 --- /dev/null +++ b/python/pytests/golden/coalesce_test/test_coalesce_three.jsonl @@ -0,0 +1,6 @@ +{"_time":"1996-12-19T16:39:57.000000000","_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0,"o":15,"coalesced_val":5} +{"_time":"1996-12-19T16:39:58.000000000","_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0,"o":15,"coalesced_val":24} +{"_time":"1996-12-19T16:39:59.000000000","_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0,"o":15,"coalesced_val":17} +{"_time":"1996-12-19T16:40:00.000000000","_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0,"o":15,"coalesced_val":9} +{"_time":"1996-12-19T16:40:01.000000000","_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null,"o":15,"coalesced_val":12} +{"_time":"1996-12-19T16:40:02.000000000","_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null,"o":15,"coalesced_val":15} From 3826664819b93f8b36dec21afa76bf6a9fdd553a Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Tue, 22 Aug 2023 11:43:59 -0500 Subject: [PATCH 15/18] code review comments --- python/pytests/clamp_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pytests/clamp_test.py b/python/pytests/clamp_test.py index 9e1307fdb..86e2f9a39 100644 --- a/python/pytests/clamp_test.py +++ b/python/pytests/clamp_test.py @@ -50,7 +50,7 @@ def banking_source() -> kd.sources.CsvString: return kd.sources.CsvString(content, time_column_name="time", key_column_name="key") -def test_clamp_banknig_min_max(banking_source, golden) -> None: +def test_clamp_banking_min_max(banking_source, golden) -> None: current_balance = banking_source.col("current_balance") min_balance = banking_source.col("min_balance") max_balance = banking_source.col("max_balance") From 6001d32a3722985f096ff048eabd7d3526fac635 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Tue, 22 Aug 2023 11:54:22 -0500 Subject: [PATCH 16/18] linting --- python/pysrc/kaskada/_timestream.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 679e79f2d..285846b01 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -356,7 +356,8 @@ def __rmul__(self, lhs: Union[Timestream, Literal]) -> Timestream: def powf(self, power: Union[Timestream, Literal]) -> Timestream: """ - + Create a Timestream raising `this` to the power of `power`. + Parameters ---------- power : Union[Timestream, Literal] From d181a44206f019734e2480f70b21b4e9619ba961 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Tue, 22 Aug 2023 12:03:10 -0500 Subject: [PATCH 17/18] linting --- python/pysrc/kaskada/_timestream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 285846b01..0d5da806c 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -357,7 +357,7 @@ def __rmul__(self, lhs: Union[Timestream, Literal]) -> Timestream: def powf(self, power: Union[Timestream, Literal]) -> Timestream: """ Create a Timestream raising `this` to the power of `power`. - + Parameters ---------- power : Union[Timestream, Literal] From 9c89ed5ac89737af53fdd28019efee54d4c9c840 Mon Sep 17 00:00:00 2001 From: Kevin J Nguyen Date: Tue, 22 Aug 2023 12:07:29 -0500 Subject: [PATCH 18/18] rust audits advisories --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b96fd875..9f2a6dc3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3908,9 +3908,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.1" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted",