From 9624f8f27a83e0c4253f38738658ea180682c888 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 17:50:48 -0600 Subject: [PATCH 01/12] adds another test for closing connections --- src/tests/test_magic.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index c41246504..e9d445535 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -6,6 +6,7 @@ import pytest from sqlalchemy import create_engine +from sql.connection import Connection from conftest import runsql @@ -368,7 +369,7 @@ def test_json_in_select(ip): assert result == [('{"greeting": "Farewell sweet {person}"}',)] -def test_close_connection(ip): +def test_closed_connections_are_no_longer_listed(ip): connections = runsql(ip, "%sql -l") connection_name = list(connections)[0] runsql(ip, f"%sql -x {connection_name}") @@ -376,6 +377,19 @@ def test_close_connection(ip): assert connection_name not in connections_afterward +def test_close_connection(ip, tmp_empty): + # open two connections + ip.run_cell("%sql sqlite:///one.db") + ip.run_cell("%sql sqlite:///two.db") + + # close them + ip.run_cell("%sql -x sqlite:///one.db") + ip.run_cell("%sql --close sqlite:///two.db") + + assert "sqlite:///one.db" not in Connection.connections + assert "sqlite:///two.db" not in Connection.connections + + def test_pass_existing_engine(ip, tmp_empty): ip.user_global_ns["my_engine"] = create_engine("sqlite:///my.db") ip.run_line_magic("sql", " my_engine ") From cbe8f54c12ff4dc415d31fd715c6089c7af9e664 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 18:21:57 -0600 Subject: [PATCH 02/12] adds test to ensure pwd is not displayed when listing connections --- src/sql/connection.py | 2 +- src/tests/test_connection.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 src/tests/test_connection.py diff --git a/src/sql/connection.py b/src/sql/connection.py index d64e27edc..b7b83b3e4 100644 --- a/src/sql/connection.py +++ b/src/sql/connection.py @@ -140,7 +140,7 @@ def connection_list(cls): template = " * {}" else: template = " {}" - result.append(template.format(engine_url.__repr__())) + result.append(template.format(repr(engine_url))) return "\n".join(result) @classmethod diff --git a/src/tests/test_connection.py b/src/tests/test_connection.py new file mode 100644 index 000000000..2137f85a9 --- /dev/null +++ b/src/tests/test_connection.py @@ -0,0 +1,12 @@ +from unittest.mock import Mock + +from sqlalchemy.engine import Engine + +from sql.connection import Connection + + +def test_password_isnt_displayed(monkeypatch): + monkeypatch.setattr(Engine, "connect", Mock()) + Connection.from_connect_str("postgresql://user:topsecret@somedomain.com/db") + + assert "topsecret" not in Connection.connection_list() From 12a693699d60f549ff2ea9aceaa076e03e19a832 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 18:34:40 -0600 Subject: [PATCH 03/12] adds xfail test --- src/tests/test_magic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index e9d445535..107fd437e 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -1,3 +1,4 @@ +from pathlib import Path import os.path import re import tempfile @@ -390,6 +391,13 @@ def test_close_connection(ip, tmp_empty): assert "sqlite:///two.db" not in Connection.connections +@pytest.mark.xfail(reason="known parse @ parser.py error") +def test_sqlite_path_with_spaces(ip, tmp_empty): + ip.run_cell("%sql sqlite:///some database.db") + + assert Path("some database.db").is_file() + + def test_pass_existing_engine(ip, tmp_empty): ip.user_global_ns["my_engine"] = create_engine("sqlite:///my.db") ip.run_line_magic("sql", " my_engine ") From 8c77f7c9c2f879f71aae50b52beef3ec91e7c321 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 18:44:28 -0600 Subject: [PATCH 04/12] adds example for connection to a sqlite db with spaces - closes #35 --- doc/howto.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/doc/howto.md b/doc/howto.md index de34d723a..4828eb9a2 100644 --- a/doc/howto.md +++ b/doc/howto.md @@ -110,3 +110,42 @@ Pass existing engines to `%sql` %%sql SELECT MYSUM(1, 2) ``` + +## Connect to a SQLite database with spaces + +Currently, due to a limitation in the argument parser, it's not possible to directly connect to SQLite databases whose path contains spaces; however, you can do it by creating the engine first. + +### Setup + +```{code-cell} ipython3 +%pip install jupysql --quiet +``` + +```{code-cell} ipython3 +%load_ext sql +``` + +## Connect to db + +```{code-cell} ipython3 +from sqlalchemy import create_engine + +engine = create_engine("sqlite:///my database.db") +``` + +Add some sample data: + +```{code-cell} ipython3 +import pandas as pd + +_ = pd.DataFrame({"x": range(5)}).to_sql("numbers", engine) +``` + +```{code-cell} ipython3 +%sql engine +``` + +```{code-cell} ipython3 +%%sql +SELECT * FROM numbers +``` From 410322dfebcc96bfc9fa0e47cc414a4674f84389 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 18:54:32 -0600 Subject: [PATCH 05/12] documents to to pass credentials securely - closes #40 --- doc/connecting.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/doc/connecting.md b/doc/connecting.md index 76e223daa..220599f19 100644 --- a/doc/connecting.md +++ b/doc/connecting.md @@ -51,6 +51,39 @@ a flag with (-a|--connection_arguments)the connection string as a JSON string. S %sql -a '{"timeout":10, "mode":"ro"}' sqlite:// SELECT * from work; ``` ++++ + +## Connecting securely + +**It is highly recommended** that you do not pass plain credentials. + +```{code-cell} ipython3 +%load_ext sql +``` + +One option is to use `getpass`, type your password, build your connection string and pass it to `%sql`: + +```{code-cell} ipython3 +from getpass import getpass +``` + +```python +password = getpass() +connection_string = f"postgresql://user:{password}@localhost/database" +%sql $connection_string +``` + ++++ + +You may also set the `DATABASE_URL` environment variable, and `%sql` will automatically load it from there: + +```python +# without any args, %sql reads from DATABASE_URL +%sql +``` + ++++ + ## DSN connections Alternately, you can store connection info in a configuration file, under a section name chosen to refer to your database. From 6d29c2c415282891493f25c535133f451b8c0554 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 19:01:11 -0600 Subject: [PATCH 06/12] improving autolimit and autodisplay tests --- src/tests/test_magic.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index 107fd437e..982054f8d 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -181,9 +181,11 @@ def test_connection_args_double_quotes(ip): # assert 'Shakespeare' in str(persisted) -def test_displaylimit(ip): +@pytest.mark.parametrize("value", ["None", "0"]) +def test_displaylimit_disabled(ip, value): ip.run_line_magic("config", "SqlMagic.autolimit = None") - ip.run_line_magic("config", "SqlMagic.displaylimit = None") + + ip.run_line_magic("config", f"SqlMagic.displaylimit = {value}") result = runsql( ip, "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) " @@ -202,6 +204,19 @@ def test_displaylimit(ip): assert "cherry" not in result._repr_html_() +def test_displaylimit(ip): + ip.run_line_magic("config", "SqlMagic.autolimit = None") + + ip.run_line_magic("config", "SqlMagic.displaylimit = 1") + result = runsql( + ip, + "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) " + "AS Result ORDER BY 1;", + ) + assert "apple" in result._repr_html_() + assert "cherry" not in result._repr_html_() + + def test_column_local_vars(ip): ip.run_line_magic("config", "SqlMagic.column_local_vars = True") result = runsql(ip, "SELECT * FROM author;") @@ -418,9 +433,17 @@ def test_pass_existing_engine(ip, tmp_empty): # theres some weird shared state with this one, moving it to the end def test_autolimit(ip): + # test table has two rows ip.run_line_magic("config", "SqlMagic.autolimit = 0") result = runsql(ip, "SELECT * FROM test;") assert len(result) == 2 + + # test table has two rows + ip.run_line_magic("config", "SqlMagic.autolimit = None") + result = runsql(ip, "SELECT * FROM test;") + assert len(result) == 2 + + # test setting autolimit to 1 ip.run_line_magic("config", "SqlMagic.autolimit = 1") result = runsql(ip, "SELECT * FROM test;") assert len(result) == 1 From f83f5ce76c0b1c26f2f916ffe581664af0706788 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 19:02:23 -0600 Subject: [PATCH 07/12] cleans up test --- src/tests/test_magic.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index 982054f8d..30afe32f1 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -194,14 +194,6 @@ def test_displaylimit_disabled(ip, value): assert "apple" in result._repr_html_() assert "banana" in result._repr_html_() assert "cherry" in result._repr_html_() - ip.run_line_magic("config", "SqlMagic.displaylimit = 1") - result = runsql( - ip, - "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) " - "AS Result ORDER BY 1;", - ) - assert "apple" in result._repr_html_() - assert "cherry" not in result._repr_html_() def test_displaylimit(ip): From b51d9e332ac501634d20346a17a3bba865d4e7fd Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 19:06:20 -0600 Subject: [PATCH 08/12] cleaning up tests --- src/tests/test_magic.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index 30afe32f1..4dcc790e0 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -186,27 +186,20 @@ def test_displaylimit_disabled(ip, value): ip.run_line_magic("config", "SqlMagic.autolimit = None") ip.run_line_magic("config", f"SqlMagic.displaylimit = {value}") - result = runsql( - ip, - "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) " - "AS Result ORDER BY 1;", - ) - assert "apple" in result._repr_html_() - assert "banana" in result._repr_html_() - assert "cherry" in result._repr_html_() + result = runsql(ip, "SELECT * FROM author;") + + assert "Brecht" in result._repr_html_() + assert "Shakespeare" in result._repr_html_() def test_displaylimit(ip): ip.run_line_magic("config", "SqlMagic.autolimit = None") ip.run_line_magic("config", "SqlMagic.displaylimit = 1") - result = runsql( - ip, - "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) " - "AS Result ORDER BY 1;", - ) - assert "apple" in result._repr_html_() - assert "cherry" not in result._repr_html_() + result = runsql(ip, "SELECT * FROM author ORDER BY first_name;") + + assert "Brecht" in result._repr_html_() + assert "Shakespeare" not in result._repr_html_() def test_column_local_vars(ip): From 0ea506fb3a00d174fd55fb76d0a28a32376f1606 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 19:11:20 -0600 Subject: [PATCH 09/12] testing column names are visible. closes #50 --- src/tests/conftest.py | 1 + src/tests/test_magic.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 18480da47..a743eabdb 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -55,6 +55,7 @@ def ip(): "CREATE TABLE author (first_name, last_name, year_of_death)", "INSERT INTO author VALUES ('William', 'Shakespeare', 1616)", "INSERT INTO author VALUES ('Bertold', 'Brecht', 1956)", + "CREATE TABLE empty_table (column INT, another INT)", ], ) yield ip_session diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index 4dcc790e0..fe5850a2a 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -391,6 +391,13 @@ def test_close_connection(ip, tmp_empty): assert "sqlite:///two.db" not in Connection.connections +def test_column_names_visible(ip, tmp_empty): + res = ip.run_line_magic("sql", "SELECT * FROM empty_table") + + assert "column" in res._repr_html_() + assert "another" in res._repr_html_() + + @pytest.mark.xfail(reason="known parse @ parser.py error") def test_sqlite_path_with_spaces(ip, tmp_empty): ip.run_cell("%sql sqlite:///some database.db") From c666007a0f95f4eabf7b186d16ecf84af3ad1814 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 19:16:44 -0600 Subject: [PATCH 10/12] testing << operator - closes #44 --- src/tests/test_command.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/test_command.py b/src/tests/test_command.py index 487725dc5..808f5c9f1 100644 --- a/src/tests/test_command.py +++ b/src/tests/test_command.py @@ -18,6 +18,7 @@ def sql_magic(ip): ("sqlite://", "", "", "sqlite://", None), ("SELECT * FROM TABLE", "", "SELECT * FROM TABLE\n", "", None), ("SELECT * FROM", "TABLE", "SELECT * FROM\nTABLE", "", None), + ("my_var << SELECT * FROM table", "", "SELECT * FROM table\n", "", "my_var"), ("my_var << SELECT *", "FROM table", "SELECT *\nFROM table", "", "my_var"), ("[db]", "", "", "sqlite://", None), ], @@ -26,7 +27,8 @@ def sql_magic(ip): "connection-string", "sql-query", "sql-query-in-line-and-cell", - "parsed-var", + "parsed-var-single-line", + "parsed-var-multi-line", "config", ], ) From f8c9d1b66bebf3ba36cde3bfcf4aed58c9c23802 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Fri, 30 Dec 2022 19:21:36 -0600 Subject: [PATCH 11/12] mocking psycopg2 --- src/tests/test_connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tests/test_connection.py b/src/tests/test_connection.py index 2137f85a9..22bd009f8 100644 --- a/src/tests/test_connection.py +++ b/src/tests/test_connection.py @@ -1,3 +1,4 @@ +import sys from unittest.mock import Mock from sqlalchemy.engine import Engine @@ -6,7 +7,9 @@ def test_password_isnt_displayed(monkeypatch): + monkeypatch.setitem(sys.modules, "psycopg2", Mock()) monkeypatch.setattr(Engine, "connect", Mock()) + Connection.from_connect_str("postgresql://user:topsecret@somedomain.com/db") assert "topsecret" not in Connection.connection_list() From 647bea704d374e7e5edaa6b4e85a47d148721ae7 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Mon, 2 Jan 2023 13:17:19 -0600 Subject: [PATCH 12/12] adds inline env var example --- doc/connecting.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/doc/connecting.md b/doc/connecting.md index 220599f19..b0cc6a078 100644 --- a/doc/connecting.md +++ b/doc/connecting.md @@ -61,13 +61,15 @@ a flag with (-a|--connection_arguments)the connection string as a JSON string. S %load_ext sql ``` +### Building connection string + One option is to use `getpass`, type your password, build your connection string and pass it to `%sql`: -```{code-cell} ipython3 -from getpass import getpass -``` ++++ ```python +from getpass import getpass + password = getpass() connection_string = f"postgresql://user:{password}@localhost/database" %sql $connection_string @@ -75,7 +77,19 @@ connection_string = f"postgresql://user:{password}@localhost/database" +++ -You may also set the `DATABASE_URL` environment variable, and `%sql` will automatically load it from there: +### Using `DATABASE_URL` + ++++ + +You may also set the `DATABASE_URL` environment variable, and `%sql` will automatically load it from there. You can do it either by setting the environment variable from your terminal or in your notebook: + +```python +from getpass import getpass +from os import environ + +password = getpass() +environ["DATABASE_URL"] = f"postgresql://user:{password}@localhost/database" +``` ```python # without any args, %sql reads from DATABASE_URL