docs: blog for the 1 billion row challenge

docs/posts/1brc/.gitignore

@@ -0,0 +1 @@
+1brc

docs/posts/1brc/index.qmd

@@ -0,0 +1,111 @@
+---
+title: "1 billion row challenge with Ibis and DuckDB"
+author: ""
+date: "2024-01-40"
+categories:
+    - blog
+    - duckdb
+---
+
+## Overview
+
+This is a redux of [The One Billion Row Challenge](https://www.morling.dev/blog/one-billion-row-challenge/),
+
+https://github.com/gunnarmorling/1brc
+
+```{.bash}
+gh repo clone gunnarmorling/1brc
+```
+
+```{.bash}
+cd 1brc/src/main/python
+python create_measurements.py 1_000_000_000
+```
+
+```{.python}
+import ibis
+import polars as pl
+import pyarrow as pa
+
+#ibis.set_backend("polars")
+#ibis.set_backend("duckdb")
+ibis.set_backend("datafusion")
+ibis.options.interactive = True
+```
+
+```{.python}
+duckdb_kwargs = {
+    "delim": ";",
+    "header": False,
+    "columns": {"station": "VARCHAR", "temperature": "DOUBLE"},
+}
+
+polars_kwargs = {
+    "separator": ";",
+    "has_header": False,
+    "new_columns": ["station", "temperature"],
+    "schema": {"station": pl.Utf8, "temperature": pl.Float64},
+}
+
+datafusion_kwargs = {
+    "delimiter": ";",
+    "has_header": False,
+    "schema": pa.schema(
+        [
+            (
+                "station",
+                pa.string(),
+            ),
+            (
+                "temperature",
+                pa.float32(),
+            ),
+        ]
+    ),
+    "file_extension": ".txt",
+}
+
+clickhouse_kwargs = {
+    "format": "CSV",
+    "types": {"station": "String", "temperature": "Float64"},
+}
+
+# kwargs = duckdb_kwargs if ibis.get_backend().name == "duckdb" else polars_kwargs
+match ibis.get_backend().name:
+    case "duckdb":
+        kwargs = duckdb_kwargs
+    case "polars":
+        kwargs = polars_kwargs
+    case "datafusion":
+        kwargs = datafusion_kwargs
+
+
+kwargs
+```
+
+```{.python}
+t = ibis.read_csv("1brc/data/measurements.txt", **kwargs)
+t
+```
+
+```{.python}
+f"{t.count().to_pandas():,}"
+```
+
+```{python}
+import time
+
+t1 = time.time()
+res = (
+    t.group_by(ibis._.station)
+    .agg(
+        min_temp=ibis._.temperature.min(),
+        mean_temp=ibis._.temperature.mean(),
+        max_temp=ibis._.temperature.max(),
+    )
+    .order_by(ibis._.station.desc())
+)
+print(res)
+t2 = time.time()
+t2 - t1
+```