diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b5f0707e..7cb2138a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ * [API Change] Drops support for old versions of IPython (removed imports from `IPython.utils.traitlets`) * [Feature] Adds `%%config SqlMagic.autopolars = True` ([#138](https://github.com/ploomber/jupysql/issues/138)) +* [Feature] Support new variable substitution as {{a}} format ([#137](https://github.com/ploomber/jupysql/pull/137)) + ## 0.5.6 (2023-02-16) * [Feature] Shows missing driver package suggestion message ([#124](https://github.com/ploomber/jupysql/issues/124)) diff --git a/doc/_toc.yml b/doc/_toc.yml index 1fa9fc1b7..215c1b48d 100644 --- a/doc/_toc.yml +++ b/doc/_toc.yml @@ -13,6 +13,7 @@ parts: - file: compose - file: user-guide/tables-columns - file: plot-legacy + - file: user-guide/template - caption: Integrations chapters: diff --git a/doc/community/vs.md b/doc/community/vs.md index 06aac9ff6..808774a4d 100644 --- a/doc/community/vs.md +++ b/doc/community/vs.md @@ -6,4 +6,5 @@ JupySQL is an actively maintained fork of [ipython-sql](https://github.com/cathe If you're migrating from `ipython-sql` to JupySQL, these are the differences (it most cases, no code changes are needed): -- Since `0.6` JupySQL no longer supports old versions of IPython \ No newline at end of file +- Since `0.6` JupySQL no longer supports old versions of IPython +- Variable expansion is being replaced from `{variable}`, `${variable}` to `{{variable}}` \ No newline at end of file diff --git a/doc/intro.md b/doc/intro.md index 350e492de..e39597cf8 100644 --- a/doc/intro.md +++ b/doc/intro.md @@ -124,6 +124,10 @@ or a single dictionary with a tuple of scalar values per key (``result.dict()``) ## Variable substitution +```{versionchanged} 0.5.7 +This is a legacy API that's kept for backwards compatibility. +``` + Bind variables (bind parameters) can be used in the "named" (:x) style. The variable names used should be defined in the local namespace. @@ -166,6 +170,8 @@ can be used in multi-line ``%%sql``: FROM languages ``` ++++ + ## Considerations Because jupysql accepts `--`-delimited options like `--persist`, but `--` diff --git a/doc/user-guide/template.md b/doc/user-guide/template.md new file mode 100644 index 000000000..03bef4d9a --- /dev/null +++ b/doc/user-guide/template.md @@ -0,0 +1,52 @@ +--- +jupytext: + text_representation: + extension: .md + format_name: myst + format_version: 0.13 + jupytext_version: 1.14.5 +kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +# Template + +## Variable Expansion as `{{variable}}` + +We support the variable expansion in the form of `{{variable}}`, this also allows the user to write the query as template with some dynamic variables + +```{code-cell} ipython3 +:tags: [remove-cell] + +%load_ext sql +from pathlib import Path +from urllib.request import urlretrieve + +if not Path("penguins.csv").is_file(): + urlretrieve( + "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/penguins.csv", + "penguins.csv", + ) +%sql duckdb:// +``` + +Now, let's give a simple query template and define some variables where we will apply in the template: + +```{code-cell} ipython3 +dynamic_limit = 5 +dynamic_column = "island, sex" +``` + +```{code-cell} ipython3 +%sql SELECT {{dynamic_column}} FROM penguins.csv LIMIT {{dynamic_limit}} +``` + +Note that variables will be fetched from the local namespace into the SQL statement. + +Please aware that we also support the `$variable` or `{variable_name}` way, but those will be deprecated in future version, [see more](https://jupysql.ploomber.io/en/latest/intro.html?highlight=variable#variable-substitution). + +```{code-cell} ipython3 + +``` diff --git a/src/sql/command.py b/src/sql/command.py index 319f0b646..920a1675a 100644 --- a/src/sql/command.py +++ b/src/sql/command.py @@ -1,4 +1,7 @@ +import re +import warnings from IPython.core.magic_arguments import parse_argstring +from jinja2 import Template from sqlalchemy.engine import Engine @@ -19,12 +22,9 @@ class SQLCommand: """ def __init__(self, magic, user_ns, line, cell) -> None: - # Parse variables (words wrapped in {}) for %%sql magic - # (for %sql this is done automatically) - cell = magic.shell.var_expand(cell) - + # Support for the variable substition in the SQL clause + line, cell = self._var_expand(magic, user_ns, line, cell) self.args = parse.magic_args(magic.execute, line) - # self.args.line (everything that appears after %sql/%%sql in the first line) # is splited in tokens (delimited by spaces), this checks if we have one arg one_arg = len(self.args.line) == 1 @@ -45,7 +45,6 @@ def __init__(self, magic, user_ns, line, cell) -> None: add_alias = True else: add_alias = False - self.command_text = " ".join(line_for_command) + "\n" + cell if self.args.file: @@ -89,3 +88,29 @@ def connection(self): def result_var(self): """Returns the result_var""" return self.parsed["result_var"] + + def _var_expand(self, magic, user_ns, line, cell): + """ + Support for the variable substition in the SQL clause + For now, we have enabled two ways: + 1. Latest format, {{a}}, we use jinja2 to parse the string with {{a}} format + 2. Legacy format, {a}, $a, and :a format. + + We will deprecate the legacy format feature in next major version + """ + self.is_legacy_var_expand_parsed = False + # Latest format parsing + # TODO: support --param and --use-global logic here + # Ref: https://github.com/ploomber/jupysql/issues/93 + line = Template(line).render(user_ns) + cell = Template(cell).render(user_ns) + # Legacy format parsing + parsed_cell = magic.shell.var_expand(cell, depth=2) + parsed_line = magic.shell.var_expand(line, depth=2) + # Exclusive the string with "://", but has :variable + has_SQLAlchemy_var_expand = re.search("(?