Adds support for parameterizing queries using {{variable}} (#137)

CHANGELOG.md

@@ -8,6 +8,8 @@
 * [API Change] Drops support for old versions of IPython (removed imports from `IPython.utils.traitlets`)
 * [Feature] Adds `%%config SqlMagic.autopolars = True` ([#138](https://github.com/ploomber/jupysql/issues/138))
 
+* [Feature] Support new variable substitution as {{a}} format ([#137](https://github.com/ploomber/jupysql/pull/137))
+
 ## 0.5.6 (2023-02-16)
 
 * [Feature] Shows missing driver package suggestion message ([#124](https://github.com/ploomber/jupysql/issues/124))

doc/_toc.yml

@@ -13,6 +13,7 @@ parts:
     - file: compose
     - file: user-guide/tables-columns
     - file: plot-legacy
+    - file: user-guide/template
 
   - caption: Integrations
     chapters:

doc/community/vs.md

@@ -6,4 +6,5 @@ JupySQL is an actively maintained fork of [ipython-sql](https://github.com/cathe
 
 If you're migrating from `ipython-sql` to JupySQL, these are the differences (it most cases, no code changes are needed):
 
-- Since `0.6` JupySQL no longer supports old versions of IPython
+- Since `0.6` JupySQL no longer supports old versions of IPython
+- Variable expansion is being replaced from `{variable}`, `${variable}` to `{{variable}}`

doc/intro.md

@@ -124,6 +124,10 @@ or a single dictionary with a tuple of scalar values per key (``result.dict()``)
 
 ## Variable substitution 
 
+```{versionchanged} 0.5.7
+This is a legacy API that's kept for backwards compatibility.
+```
+
 Bind variables (bind parameters) can be used in the "named" (:x) style.
 The variable names used should be defined in the local namespace.
 
@@ -166,6 +170,8 @@ can be used in multi-line ``%%sql``:
 FROM languages
 ```
 
++++
+
 ## Considerations
 
 Because jupysql accepts `--`-delimited options like `--persist`, but `--` 

doc/user-guide/template.md

@@ -0,0 +1,52 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.14.5
+kernelspec:
+  display_name: Python 3 (ipykernel)
+  language: python
+  name: python3
+---
+
+# Template
+
+## Variable Expansion as `{{variable}}`
+
+We support the variable expansion in the form of `{{variable}}`, this also allows the user to write the query as template with some dynamic variables
+
+```{code-cell} ipython3
+:tags: [remove-cell]
+
+%load_ext sql
+from pathlib import Path
+from urllib.request import urlretrieve
+
+if not Path("penguins.csv").is_file():
+    urlretrieve(
+        "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/penguins.csv",
+        "penguins.csv",
+    )
+%sql duckdb://
+```
+
+Now, let's give a simple query template and define some variables where we will apply in the template:
+
+```{code-cell} ipython3
+dynamic_limit = 5
+dynamic_column = "island, sex"
+```
+
+```{code-cell} ipython3
+%sql SELECT {{dynamic_column}} FROM penguins.csv LIMIT {{dynamic_limit}}
+```
+
+Note that variables will be fetched from the local namespace into the SQL statement.
+
+Please aware that we also support the `$variable` or `{variable_name}` way, but those will be deprecated in future version, [see more](https://jupysql.ploomber.io/en/latest/intro.html?highlight=variable#variable-substitution).
+
+```{code-cell} ipython3
+
+```

src/sql/command.py

@@ -1,4 +1,7 @@
+import re
+import warnings
 from IPython.core.magic_arguments import parse_argstring
+from jinja2 import Template
 
 from sqlalchemy.engine import Engine
 
@@ -19,12 +22,9 @@ class SQLCommand:
     """
 
     def __init__(self, magic, user_ns, line, cell) -> None:
-        # Parse variables (words wrapped in {}) for %%sql magic
-        # (for %sql this is done automatically)
-        cell = magic.shell.var_expand(cell)
-
+        # Support for the variable substition in the SQL clause
+        line, cell = self._var_expand(magic, user_ns, line, cell)
         self.args = parse.magic_args(magic.execute, line)
-
         # self.args.line (everything that appears after %sql/%%sql in the first line)
         # is splited in tokens (delimited by spaces), this checks if we have one arg
         one_arg = len(self.args.line) == 1
@@ -45,7 +45,6 @@ def __init__(self, magic, user_ns, line, cell) -> None:
             add_alias = True
         else:
             add_alias = False
-
         self.command_text = " ".join(line_for_command) + "\n" + cell
 
         if self.args.file:
@@ -89,3 +88,29 @@ def connection(self):
     def result_var(self):
         """Returns the result_var"""
         return self.parsed["result_var"]
+
+    def _var_expand(self, magic, user_ns, line, cell):
+        """
+        Support for the variable substition in the SQL clause
+        For now, we have enabled two ways:
+        1. Latest format, {{a}}, we use jinja2 to parse the string with {{a}} format
+        2. Legacy format, {a}, $a, and :a format.
+
+        We will deprecate the legacy format feature in next major version
+        """
+        self.is_legacy_var_expand_parsed = False
+        # Latest format parsing
+        # TODO: support --param and --use-global logic here
+        # Ref: https://github.com/ploomber/jupysql/issues/93
+        line = Template(line).render(user_ns)
+        cell = Template(cell).render(user_ns)
+        # Legacy format parsing
+        parsed_cell = magic.shell.var_expand(cell, depth=2)
+        parsed_line = magic.shell.var_expand(line, depth=2)
+        # Exclusive the string with "://", but has :variable
+        has_SQLAlchemy_var_expand = re.search("(?<!://):[^/]+", line) or ":" in cell
+        if parsed_line != line or parsed_cell != cell or has_SQLAlchemy_var_expand:
+            self.is_legacy_var_expand_parsed = True
+            warnings.warn("Please aware the variable substition. Use {{a}} instead"
+                          , FutureWarning)
+        return parsed_line, parsed_cell

src/sql/magic.py

@@ -7,6 +7,7 @@
     line_magic,
     magics_class,
     needs_local_scope,
+    no_var_expand,
 )
 from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
 from sqlalchemy.exc import OperationalError, ProgrammingError, DatabaseError
@@ -19,6 +20,7 @@
 from sql.magic_plot import SqlPlotMagic
 from sql.magic_cmd import SqlCmdMagic
 
+
 from traitlets.config.configurable import Configurable
 from traitlets import Bool, Int, Unicode, observe
 
@@ -134,6 +136,7 @@ def _mutex_autopandas_autopolars(self, change):
                 setattr(self, other, False)
                 print(f"Disabled '{other}' since '{change['name']}' was enabled.")
 
+    @no_var_expand
     @needs_local_scope
     @line_magic("sql")
     @cell_magic("sql")

src/tests/test_command.py

@@ -170,7 +170,47 @@ def test_parse_sql_when_passing_engine(ip, sql_magic, tmp_empty, line):
     assert cmd.sql_original == sql_expected
 
 
-def test_variable_substitution_cell_magic(ip, sql_magic):
+def test_variable_substitution_legacy_warning_message_dollar_prefix(
+    ip, sql_magic, capsys
+):
+    with pytest.warns(FutureWarning):
+        ip.user_global_ns["limit_number"] = 1
+        ip.run_cell_magic(
+            "sql",
+            "",
+            """
+            SELECT * FROM author LIMIT $limit_number
+            """,
+        )
+
+
+def test_variable_substitution_legacy_warning_message_single_curly(
+    ip, sql_magic, capsys
+):
+    with pytest.warns(FutureWarning):
+        ip.user_global_ns["limit_number"] = 1
+        ip.run_cell_magic(
+            "sql",
+            "",
+            """
+            SELECT * FROM author LIMIT {limit_number}
+            """,
+        )
+
+
+def test_variable_substitution_legacy_warning_message_colon(ip, sql_magic, capsys):
+    with pytest.warns(FutureWarning):
+        ip.user_global_ns["limit_number"] = 1
+        ip.run_cell_magic(
+            "sql",
+            "",
+            """
+            SELECT * FROM author LIMIT :limit_number
+            """,
+        )
+
+
+def test_variable_substitution_legacy_dollar_prefix_cell_magic(ip, sql_magic):
     ip.user_global_ns["username"] = "some-user"
 
     cmd = SQLCommand(
@@ -181,3 +221,44 @@ def test_variable_substitution_cell_magic(ip, sql_magic):
     )
 
     assert cmd.parsed["sql"] == "\nGRANT CONNECT ON DATABASE postgres TO some-user;"
+
+
+def test_variable_substitution_legacy_single_curly_cell_magic(ip, sql_magic):
+    ip.user_global_ns["username"] = "some-user"
+
+    cmd = SQLCommand(
+        sql_magic,
+        ip.user_ns,
+        line="",
+        cell="GRANT CONNECT ON DATABASE postgres TO {username};",
+    )
+
+    assert cmd.parsed["sql"] == "\nGRANT CONNECT ON DATABASE postgres TO some-user;"
+
+
+def test_variable_substitution_double_curly_cell_magic(ip, sql_magic):
+    ip.user_global_ns["username"] = "some-user"
+
+    cmd = SQLCommand(
+        sql_magic,
+        ip.user_ns,
+        line="",
+        cell="GRANT CONNECT ON DATABASE postgres TO {{username}};",
+    )
+
+    print("cmd.parsed['sql']", cmd.parsed["sql"])
+    assert cmd.parsed["sql"] == "\nGRANT CONNECT ON DATABASE postgres TO some-user;"
+
+
+def test_variable_substitution_double_curly_line_magic(ip, sql_magic):
+    ip.user_global_ns["limit_number"] = 5
+    ip.user_global_ns["column_name"] = "first_name"
+    cmd = SQLCommand(
+        sql_magic,
+        ip.user_ns,
+        line="SELECT {{column_name}} FROM author LIMIT {{limit_number}};",
+        cell="",
+    )
+
+    # print ("cmd.parsed['sql']", cmd.parsed["sql"])
+    assert cmd.parsed["sql"] == "SELECT first_name FROM author LIMIT 5;\n"