perf(sql): prevent sqlglot from extensive deepcopying every time we c…

…reate a sqlglot object (#8592)

Still profiling and adding more `copy=False` options, apparently it
greatly improves the performance.

According to profiling the recursive generation of sqlglot is still a
bottleneck for big queries which cannot be fixed on the ibis side. There
could be one option though to compile the fragments in a greedy fashion
which are going to be cached by `Node.map()` and inject those as
arbitrary strings to other sqlglot expressions.

Now it uses 30% of the memory it used before. Apparently there are still
a lot of sqlglot Literal objects around, looking into that.

fixes #8484

---------

Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com>

ibis/backends/bigquery/compiler.py

@@ -12,7 +12,7 @@
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
 from ibis import util
-from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler, paren
+from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler
 from ibis.backends.sql.datatypes import BigQueryType, BigQueryUDFType
 from ibis.backends.sql.rewrites import (
     exclude_unsupported_window_frame_from_ops,
@@ -704,10 +704,10 @@ def visit_CountStar(self, op, *, arg, where):
         return self.f.count(STAR)
 
     def visit_Degrees(self, op, *, arg):
-        return paren(180 * arg / self.f.acos(-1))
+        return sge.paren(180 * arg / self.f.acos(-1), copy=False)
 
     def visit_Radians(self, op, *, arg):
-        return paren(self.f.acos(-1) * arg / 180)
+        return sge.paren(self.f.acos(-1) * arg / 180, copy=False)
 
     def visit_CountDistinct(self, op, *, arg, where):
         if where is not None:

ibis/backends/clickhouse/compiler.py

@@ -11,12 +11,7 @@
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
 from ibis import util
-from ibis.backends.sql.compiler import (
-    NULL,
-    STAR,
-    SQLGlotCompiler,
-    parenthesize,
-)
+from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler
 from ibis.backends.sql.datatypes import ClickHouseType
 from ibis.backends.sql.dialects import ClickHouse
 from ibis.backends.sql.rewrites import rewrite_sample_as_filter
@@ -163,11 +158,11 @@ def visit_ArrayRepeat(self, op, *, arg, times):
         return self.f.arrayFlatten(self.f.arrayMap(func, self.f.range(times)))
 
     def visit_ArraySlice(self, op, *, arg, start, stop):
-        start = parenthesize(op.start, start)
+        start = self._add_parens(op.start, start)
         start_correct = self.if_(start < 0, start, start + 1)
 
         if stop is not None:
-            stop = parenthesize(op.stop, stop)
+            stop = self._add_parens(op.stop, stop)
 
             length = self.if_(
                 stop < 0,

ibis/backends/datafusion/compiler.py

@@ -11,13 +11,7 @@
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
-from ibis.backends.sql.compiler import (
-    FALSE,
-    NULL,
-    STAR,
-    SQLGlotCompiler,
-    paren,
-)
+from ibis.backends.sql.compiler import FALSE, NULL, STAR, SQLGlotCompiler
 from ibis.backends.sql.datatypes import DataFusionType
 from ibis.backends.sql.dialects import DataFusion
 from ibis.backends.sql.rewrites import rewrite_sample_as_filter
@@ -261,7 +255,7 @@ def visit_DayOfWeekIndex(self, op, *, arg):
 
     def visit_DayOfWeekName(self, op, *, arg):
         return sg.exp.Case(
-            this=paren(self.f.date_part("dow", arg) + 6) % 7,
+            this=sge.paren(self.f.date_part("dow", arg) + 6, copy=False) % 7,
             ifs=list(starmap(self.if_, enumerate(calendar.day_name))),
         )
 
@@ -438,7 +432,7 @@ def visit_StringConcat(self, op, *, arg):
     def visit_Aggregate(self, op, *, parent, groups, metrics):
         """Support `GROUP BY` expressions in `SELECT` since DataFusion does not."""
         quoted = self.quoted
-        metrics = tuple(starmap(self._dedup_name, metrics.items()))
+        metrics = tuple(self._cleanup_names(metrics))
 
         if groups:
             # datafusion doesn't support count distinct aggregations alongside
@@ -459,7 +453,7 @@ def visit_Aggregate(self, op, *, parent, groups, metrics):
                 )
             )
             table = (
-                sg.select(*cols, *starmap(self._dedup_name, groups.items()))
+                sg.select(*cols, *self._cleanup_names(groups))
                 .from_(parent)
                 .subquery(parent.alias)
             )

ibis/backends/duckdb/compiler.py

@@ -11,7 +11,7 @@
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
-from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler, paren
+from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler
 from ibis.backends.sql.datatypes import DuckDBType
 
 _INTERVAL_SUFFIXES = {
@@ -402,6 +402,7 @@ def visit_StructField(self, op, *, arg, field):
         if not isinstance(op.arg, (ops.Field, sge.Struct)):
             # parenthesize anything that isn't a simple field access
             return sge.Dot(
-                this=paren(arg), expression=sg.to_identifier(field, quoted=self.quoted)
+                this=sge.paren(arg),
+                expression=sg.to_identifier(field, quoted=self.quoted),
             )
         return super().visit_StructField(op, arg=arg, field=field)

ibis/backends/flink/compiler.py

@@ -8,7 +8,7 @@
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
-from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler, paren
+from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler
 from ibis.backends.sql.datatypes import FlinkType
 from ibis.backends.sql.dialects import Flink
 from ibis.backends.sql.rewrites import (
@@ -467,12 +467,12 @@ def visit_TimestampBucket(self, op, *, arg, interval, offset):
         bucket_width = op.interval.value
         unit_func = self.f["dayofmonth" if unit.upper() == "DAY" else unit]
 
-        arg = self.f.anon.timestampadd(unit_var, -paren(offset), arg)
+        arg = self.f.anon.timestampadd(unit_var, -sge.paren(offset, copy=False), arg)
         mod = unit_func(arg) % bucket_width
 
         return self.f.anon.timestampadd(
             unit_var,
-            -paren(mod) + offset,
+            -sge.paren(mod, copy=False) + offset,
             self.v[f"FLOOR({arg.sql(self.dialect)} TO {unit_var.sql(self.dialect)})"],
         )
 

ibis/backends/mssql/compiler.py

@@ -15,7 +15,6 @@
     STAR,
     TRUE,
     SQLGlotCompiler,
-    paren,
 )
 from ibis.backends.sql.datatypes import MSSQLType
 from ibis.backends.sql.dialects import MSSQL
@@ -69,6 +68,7 @@ class MSSQLCompiler(SQLGlotCompiler):
         rewrite_rows_range_order_by_window,
         *SQLGlotCompiler.rewrites,
     )
+    copy_func_args = True
 
     UNSUPPORTED_OPERATIONS = frozenset(
         (
@@ -189,7 +189,7 @@ def visit_StringLength(self, op, *, arg):
 
         Thanks to @arkanovicz for this glorious hack.
         """
-        return paren(self.f.len(self.f.concat("A", arg, "Z")) - 2)
+        return sge.paren(self.f.len(self.f.concat("A", arg, "Z")) - 2, copy=False)
 
     def visit_GroupConcat(self, op, *, arg, sep, where):
         if where is not None:

ibis/backends/postgres/compiler.py

@@ -11,7 +11,7 @@
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
 import ibis.expr.rules as rlz
-from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler, paren
+from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler
 from ibis.backends.sql.datatypes import PostgresType
 from ibis.backends.sql.dialects import Postgres
 from ibis.backends.sql.rewrites import rewrite_sample_as_filter
@@ -125,7 +125,7 @@ def visit_ArgMinMax(self, op, *, arg, key, where, desc: bool):
             sge.Ordered(this=sge.Order(this=arg, expressions=[key]), desc=desc),
             where=sg.and_(*conditions),
         )
-        return paren(agg)[0]
+        return sge.paren(agg, copy=False)[0]
 
     def visit_ArgMin(self, op, *, arg, key, where):
         return self.visit_ArgMinMax(op, arg=arg, key=key, where=where, desc=False)
@@ -378,7 +378,7 @@ def visit_Modulus(self, op, *, left, right):
     def visit_RegexExtract(self, op, *, arg, pattern, index):
         pattern = self.f.concat("(", pattern, ")")
         matches = self.f.regexp_match(arg, pattern)
-        return self.if_(arg.rlike(pattern), paren(matches)[index], NULL)
+        return self.if_(arg.rlike(pattern), sge.paren(matches, copy=False)[index], NULL)
 
     def visit_FindInSet(self, op, *, needle, values):
         return self.f.coalesce(
@@ -463,7 +463,7 @@ def visit_ExtractEpochSeconds(self, op, *, arg):
 
     def visit_ArrayIndex(self, op, *, arg, index):
         index = self.if_(index < 0, self.f.cardinality(arg) + index, index)
-        return paren(arg)[index + 1]
+        return sge.paren(arg, copy=False)[index + 1]
 
     def visit_ArraySlice(self, op, *, arg, start, stop):
         neg_to_pos_index = lambda n, index: self.if_(index < 0, n + index, index)
@@ -481,7 +481,7 @@ def visit_ArraySlice(self, op, *, arg, start, stop):
             stop = neg_to_pos_index(arg_length, stop)
 
         slice_expr = sge.Slice(this=start + 1, expression=stop)
-        return paren(arg)[slice_expr]
+        return sge.paren(arg, copy=False)[slice_expr]
 
     def visit_IntervalFromInteger(self, op, *, arg, unit):
         plural = unit.plural

ibis/backends/sql/__init__.py

@@ -106,7 +106,7 @@ def _to_sqlglot(
         assert not isinstance(sql, sge.Subquery)
 
         if isinstance(sql, sge.Table):
-            sql = sg.select(STAR).from_(sql)
+            sql = sg.select(STAR, copy=False).from_(sql, copy=False)
 
         assert not isinstance(sql, sge.Subquery)
         return sql
@@ -116,7 +116,7 @@ def compile(
     ):
         """Compile an Ibis expression to a SQL string."""
         query = self._to_sqlglot(expr, limit=limit, params=params, **kwargs)
-        sql = query.sql(dialect=self.dialect, pretty=True)
+        sql = query.sql(dialect=self.dialect, pretty=True, copy=False)
         self._log(sql)
         return sql