refactor(azure_mssql): use sqlalchemy for connections [TCTC-9751]

.github/workflows/ci.yml

@@ -86,6 +86,11 @@ jobs:
         ATHENA_REGION: '${{ secrets.ATHENA_REGION }}'
         # GBQ
         GOOGLE_BIG_QUERY_CREDENTIALS: '${{ secrets.GOOGLE_BIG_QUERY_CREDENTIALS }}'
+        # Azure MSSQL
+        AZURE_MSSQL_USER: '${{ secrets.AZURE_MSSQL_USER }}'
+        AZURE_MSSQL_PASSWORD: '${{ secrets.AZURE_MSSQL_PASSWORD }}'
+        AZURE_MSSQL_HOST: '${{ secrets.AZURE_MSSQL_HOST }}'
+        AZURE_MSSQL_DATABASE: '${{ secrets.AZURE_MSSQL_DATABASE }}'
 
     - name: SonarCloud Scan
       # Only executed for one of the tested python version and pandas version

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Changed
+
+- The Azure MSSQL connector now uses `sqlalchemy` to connect to MSSQL.
+
 ## [7.3.3] 2024-11-21
 
 ### Fixed

pyproject.toml

@@ -49,6 +49,7 @@ redshift-connector = { version = "^2.0.907", optional = true }
 requests-oauthlib = { version = "2.0.0", optional = true }
 simplejson = { version = "^3.17.6", optional = true }
 snowflake-connector-python = { version = ">=2.7.12,<4.0.0", optional = true }
+sqlalchemy = { version = "^2", optional = true }
 tctc-odata = { version = ">=0.3,<1.0", optional = true }
 tenacity = { version = ">=8.0.1,<10.0.0", optional = true }
 xmltodict = { version = ">=0.13.0,<1.0", optional = true }
@@ -78,36 +79,36 @@ types-pymysql = "^1.1.0.20241103"
 
 [tool.poetry.extras]
 awsathena = ["awswrangler", "pandas"]
-azure_mssql = ["pyodbc", "pandas"]
+azure_mssql = ["pyodbc", "pandas", "sqlalchemy"]
 base = ["Authlib", "jq", "pandas", "python-slugify", "tenacity", "pandas"]
-clickhouse = ["clickhouse-driver", "pandas"]
+clickhouse = ["clickhouse-driver", "pandas", "sqlalchemy"]
 dataiku = ["dataiku-api-client", "pandas"]
 elasticsearch = ["elasticsearch", "pandas"]
 github = ["python-graphql-client", "pandas"]
 google_analytics = ["google-api-python-client", "oauth2client", "pandas"]
 google_big_query = ["google-cloud-bigquery", "pandas"]
-google_cloud_mysql = ["PyMySQL", "pandas"]
+google_cloud_mysql = ["PyMySQL", "pandas", "sqlalchemy"]
 google_sheets = ["google-api-python-client", "pandas"]
 http_api = ["oauthlib", "requests-oauthlib", "xmltodict", "pandas"]
 hubspot = ["hubspot-api-client", "pandas"]
 mongo = ["pymongo", "pandas"]
-mssql = ["pyodbc", "pandas"]
+mssql = ["pyodbc", "pandas", "sqlalchemy"]
 mssql_TLSv1_0 = ["pyodbc", "pandas"]
-mysql = ["PyMySQL", "pandas"]
+mysql = ["PyMySQL", "pandas", "sqlalchemy"]
 odata = ["oauthlib", "requests-oauthlib", "tctc_odata", "pandas"]
 odbc = ["pydobc", "pandas"]
-oracle_sql = ["cx_Oracle", "pandas"]
+oracle_sql = ["cx_Oracle", "pandas", "sqlalchemy"]
 peakina = ["peakina", "pandas"]
-postgres = ["psycopg2", "pandas"]
+postgres = ["psycopg2", "pandas", "sqlalchemy"]
 Redshift = ["redshift_connector", "lxml", "pandas"]
-sap_hana = ["pyhdb", "pandas"]
+sap_hana = ["pyhdb", "pandas", "sqlalchemy"]
 snowflake = ["snowflake-connector-python", "PyJWT", "pyarrow", "pandas"]
 soap = ["zeep", "lxml", "pandas"]
 
 # All
 all = [
-    "awswrangler",
     "Authlib",
+    "awswrangler",
     "clickhouse-driver",
     "cx-Oracle",
     "dataiku-api-client",
@@ -120,12 +121,12 @@ all = [
     "oauth2client",
     "oauthlib",
     "openpyxl",
+    "pandas",
     "peakina",
     "psycopg2",
     "pyarrow",
     "pyhdb",
     "PyJWT",
-    "pandas",
     "pymongo",
     "PyMySQL",
     "pyodbc",
@@ -135,8 +136,9 @@ all = [
     "requests-oauthlib",
     "simplejson",
     "snowflake-connector-python",
-    "tenacity",
+    "sqlalchemy",
     "tctc-odata",
+    "tenacity",
     "xmltodict",
     "zeep",
 ]
@@ -162,15 +164,17 @@ ignore_missing_imports = true
 files = [
     "toucan_connectors/auth.py",
     "toucan_connectors/awsathena/awsathena_connector.py",
+    "toucan_connectors/azure_mssql/azure_mssql_connector.py",
+    "toucan_connectors/common.py",
     "toucan_connectors/google_big_query/google_big_query_connector.py",
     "toucan_connectors/hubspot_private_app/hubspot_connector.py",
     "toucan_connectors/mongo/mongo_connector.py",
+    "toucan_connectors/mysql/mysql_connector.py",
     "toucan_connectors/peakina/peakina_connector.py",
     "toucan_connectors/postgres/postgresql_connector.py",
-    "toucan_connectors/mysql/mysql_connector.py",
+    "toucan_connectors/redshift/redshift_database_connector.py",
     "toucan_connectors/snowflake/snowflake_connector.py",
     "toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py",
-    "toucan_connectors/redshift/redshift_database_connector.py",
     "toucan_connectors/toucan_connector.py",
 ]
 

tests/azure_mssql/fixtures/world.sql

@@ -0,0 +1,59 @@
+DROP TABLE IF EXISTS City;
+CREATE TABLE City (
+        ID int,
+        Name varchar(35) NOT NULL DEFAULT '',
+        CountryCode char(3) NOT NULL DEFAULT '',
+        District varchar(20) NOT NULL DEFAULT '',
+        Population int NOT NULL DEFAULT '0'
+)
+
+INSERT INTO City VALUES (1,'Kabul','AFG','Kabol',1780000);
+INSERT INTO City VALUES (2,'Qandahar','AFG','Qandahar',237500);
+INSERT INTO City VALUES (3,'Herat','AFG','Herat',186800);
+INSERT INTO City VALUES (4,'Mazar-e-Sharif','AFG','Balkh',127800);
+INSERT INTO City VALUES (5,'Amsterdam','NLD','Noord-Holland',731200);
+INSERT INTO City VALUES (6,'Rotterdam','NLD','Zuid-Holland',593321);
+INSERT INTO City VALUES (7,'Haag','NLD','Zuid-Holland',440900);
+INSERT INTO City VALUES (8,'Utrecht','NLD','Utrecht',234323);
+INSERT INTO City VALUES (9,'Eindhoven','NLD','Noord-Brabant',201843);
+INSERT INTO City VALUES (10,'Tilburg','NLD','Noord-Brabant',193238);
+INSERT INTO City VALUES (11,'Groningen','NLD','Groningen',172701);
+INSERT INTO City VALUES (12,'Breda','NLD','Noord-Brabant',160398);
+INSERT INTO City VALUES (13,'Apeldoorn','NLD','Gelderland',153491);
+INSERT INTO City VALUES (14,'Nijmegen','NLD','Gelderland',152463);
+INSERT INTO City VALUES (15,'Enschede','NLD','Overijssel',149544);
+INSERT INTO City VALUES (16,'Haarlem','NLD','Noord-Holland',148772);
+INSERT INTO City VALUES (17,'Almere','NLD','Flevoland',142465);
+INSERT INTO City VALUES (18,'Arnhem','NLD','Gelderland',138020);
+INSERT INTO City VALUES (19,'Zaanstad','NLD','Noord-Holland',135621);
+INSERT INTO City VALUES (20,'�s-Hertogenbosch','NLD','Noord-Brabant',129170);
+INSERT INTO City VALUES (21,'Amersfoort','NLD','Utrecht',126270);
+INSERT INTO City VALUES (22,'Maastricht','NLD','Limburg',122087);
+INSERT INTO City VALUES (23,'Dordrecht','NLD','Zuid-Holland',119811);
+INSERT INTO City VALUES (24,'Leiden','NLD','Zuid-Holland',117196);
+INSERT INTO City VALUES (25,'Haarlemmermeer','NLD','Noord-Holland',110722);
+INSERT INTO City VALUES (26,'Zoetermeer','NLD','Zuid-Holland',110214);
+INSERT INTO City VALUES (27,'Emmen','NLD','Drenthe',105853);
+INSERT INTO City VALUES (28,'Zwolle','NLD','Overijssel',105819);
+INSERT INTO City VALUES (29,'Ede','NLD','Gelderland',101574);
+INSERT INTO City VALUES (30,'Delft','NLD','Zuid-Holland',95268);
+INSERT INTO City VALUES (31,'Heerlen','NLD','Limburg',95052);
+INSERT INTO City VALUES (32,'Alkmaar','NLD','Noord-Holland',92713);
+INSERT INTO City VALUES (33,'Willemstad','ANT','Cura�ao',2345);
+INSERT INTO City VALUES (34,'Tirana','ALB','Tirana',270000);
+INSERT INTO City VALUES (35,'Alger','DZA','Alger',2168000);
+INSERT INTO City VALUES (36,'Oran','DZA','Oran',609823);
+INSERT INTO City VALUES (37,'Constantine','DZA','Constantine',443727);
+INSERT INTO City VALUES (38,'Annaba','DZA','Annaba',222518);
+INSERT INTO City VALUES (39,'Batna','DZA','Batna',183377);
+INSERT INTO City VALUES (40,'S�tif','DZA','S�tif',179055);
+INSERT INTO City VALUES (41,'Sidi Bel Abb�s','DZA','Sidi Bel Abb�s',153106);
+INSERT INTO City VALUES (42,'Skikda','DZA','Skikda',128747);
+INSERT INTO City VALUES (43,'Biskra','DZA','Biskra',128281);
+INSERT INTO City VALUES (44,'Blida (el-Boulaida)','DZA','Blida',127284);
+INSERT INTO City VALUES (45,'B�ja�a','DZA','B�ja�a',117162);
+INSERT INTO City VALUES (46,'Mostaganem','DZA','Mostaganem',115212);
+INSERT INTO City VALUES (47,'T�bessa','DZA','T�bessa',112007);
+INSERT INTO City VALUES (48,'Tlemcen (Tilimsen)','DZA','Tlemcen',110242);
+INSERT INTO City VALUES (49,'B�char','DZA','B�char',107311);
+INSERT INTO City VALUES (50,'Tiaret','DZA','Tiaret',100118);

tests/azure_mssql/test_azure_mssql.py

@@ -1,39 +1,51 @@
+from os import environ
+
+import pandas as pd
+import pytest
+from pandas.testing import assert_frame_equal
+
 from toucan_connectors.azure_mssql.azure_mssql_connector import (
     AzureMSSQLConnector,
     AzureMSSQLDataSource,
 )
 
 
-def test_connection_params():
-    connector = AzureMSSQLConnector(host="my_host", user="my_user", name="")
-    params = connector.get_connection_params()
-    assert params["server"] == "my_host.database.windows.net"
-    assert params["user"] == "my_user@my_host"
-
-    connector = AzureMSSQLConnector(host="my_host.database.windows.net", user="my_user", password="", name="")
-    params = connector.get_connection_params()
-    assert params["server"] == "my_host.database.windows.net"
-    assert params["user"] == "my_user@my_host"
-
-    connector = AzureMSSQLConnector(host="my_host.database.windows.net", user="my_user@my_host", password="", name="")
-    params = connector.get_connection_params()
-    assert params["server"] == "my_host.database.windows.net"
-    assert params["user"] == "my_user@my_host"
-
-
-def test_gcmysql_get_df(mocker):
-    snock = mocker.patch("pyodbc.connect")
-    reasq = mocker.patch("pandas.read_sql")
-
-    mssql_connector = AzureMSSQLConnector(name="test", host="localhost", user="ubuntu", password="ilovetoucan")
-    ds = AzureMSSQLDataSource(domain="test", name="test", database="mssql_db", query="my_query")
-    mssql_connector.get_df(ds)
-
-    snock.assert_called_once_with(
-        server="localhost.database.windows.net",
-        user="ubuntu@localhost",
-        database="mssql_db",
-        password="ilovetoucan",
-        driver="{ODBC Driver 17 for SQL Server}",
+@pytest.fixture
+def connector() -> AzureMSSQLConnector:
+    user = environ["AZURE_MSSQL_USER"]
+    password = environ["AZURE_MSSQL_PASSWORD"]
+    host = environ["AZURE_MSSQL_HOST"]
+    return AzureMSSQLConnector(name="azure-mssql-ci", user=user, host=host, password=password, connect_timeout=3)
+
+
+@pytest.fixture
+def datasource() -> AzureMSSQLDataSource:
+    database = environ["AZURE_MSSQL_DATABASE"]
+    return AzureMSSQLDataSource(domain="azure-mssql-ci", name="Azure MSSQL CI", database=database, query="SELECT 1;")
+
+
+def test_azure_get_df_simple(connector: AzureMSSQLConnector, datasource: AzureMSSQLDataSource) -> None:
+    datasource.query = "SELECT name, population FROM City WHERE name = 'Maastricht';"
+    df = connector.get_df(datasource)
+    expected = pd.DataFrame({"name": ["Maastricht"], "population": [122_087]})
+    assert_frame_equal(df, expected)
+
+
+def test_azure_get_df_with_parameters_and_modulo(
+    connector: AzureMSSQLConnector, datasource: AzureMSSQLDataSource
+) -> None:
+    datasource.query = "SELECT * FROM City WHERE CountryCode = {{ Code }} AND Population % 1000 >= 700"
+    datasource.parameters = {"Code": "AFG"}
+    df = connector.get_df(datasource)
+    assert_frame_equal(
+        df,
+        pd.DataFrame(
+            {
+                "ID": [3, 4],
+                "Name": ["Herat", "Mazar-e-Sharif"],
+                "CountryCode": ["AFG", "AFG"],
+                "District": ["Herat", "Balkh"],
+                "Population": [186_800, 127_800],
+            }
+        ),
     )
-    reasq.assert_called_once_with("my_query", con=snock(), params=[])

tests/test_common.py

@@ -14,6 +14,7 @@
     UndefinedVariableError,
     adapt_param_type,
     apply_query_parameters,
+    convert_jinja_params_to_sqlalchemy_named,
     convert_to_numeric_paramstyle,
     convert_to_printf_templating_style,
     convert_to_qmark_paramstyle,
@@ -563,3 +564,18 @@ def test_convert_pyformat_to_numeric(query, params, expected_query, expected_ord
     converted_query, ordered_values = convert_to_numeric_paramstyle(query, params)
     assert ordered_values == expected_ordered_values
     assert converted_query == expected_query
+
+
+@pytest.mark.parametrize(
+    "query,expected",
+    [
+        ("SELECT * FROM my_table;", "SELECT * FROM my_table;"),
+        (
+            "SELECT name, population FROM City WHERE name SIMILAR TO '%aastri%' AND population >= {{min_pop}}",
+            "SELECT name, population FROM City WHERE name SIMILAR TO '%aastri%' AND population >= :min_pop",
+        ),
+    ],
+)
+def test_convert_jinja_params_to_sqlalchemy_named(query: str, expected: str) -> None:
+    result = convert_jinja_params_to_sqlalchemy_named(query)
+    assert result == expected