From 585751b61348334d73a02881fa7bd0102ee0476f Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 3 Sep 2020 13:07:28 -0700 Subject: [PATCH 1/7] add Active Directory authentication and SP --- README.md | 84 +++++++++++++++++++-------- dbt/adapters/sqlserver/connections.py | 83 ++++++++++++++++++++++++-- 2 files changed, 138 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 282855f5..bd63df89 100644 --- a/README.md +++ b/README.md @@ -15,29 +15,67 @@ Easiest install is to use pip: pip install dbt-sqlserver On Ubuntu make sure you have the ODBC header files before installing - - sudo apt install unixodbc-dev - -## Configure your profile -Configure your dbt profile for using SQL Server authentication or Integrated Security: -##### SQL Server authentication - type: sqlserver - driver: 'ODBC Driver 17 for SQL Server' (The ODBC Driver installed on your system) - server: server-host-name or ip - port: 1433 - user: username - password: password - database: databasename - schema: schemaname - -##### Integrated Security - type: sqlserver - driver: 'ODBC Driver 17 for SQL Server' - server: server-host-name or ip - port: 1433 - user: username - schema: schemaname - windows_login: True + +``` +sudo apt install unixodbc-dev +``` + +## Authentication +`SqlPassword` is the default connection method, but you can also use the following [`pyodbc`-supported ActiveDirectory methods](https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#new-andor-modified-dsn-and-connection-string-keywords) to authenticate: +- Integrated (i.e. Windows Login) +- ActiveDirectory Password +- ActiveDirectory Interactive +- ActiveDirectory Integrated +- ActiveDirectory MSI (to be implemented) +- Service Principal (a.k.a. AAD Application) +#### boilerplate +this should be in every target definition +``` +type: sqlserver +driver: 'ODBC Driver 17 for SQL Server' (The ODBC Driver installed on your system) +server: server-host-name or ip +port: 1433 +schema: schemaname +``` +#### SQL Server authentication +``` +user: username +password: password +``` + +#### Integrated Security +``` +windows_login: True +``` +#### ActiveDirectory Password +Definitely not ideal, but available +``` +authentication: ActiveDirectoryPassword +user: bill.gates@microsoft.com +password: i<3opensource? +``` +#### ActiveDirectory Interactive (*Windows only*) +brings up the Azure AD prompt so you can MFA if need be. +``` +authentication: ActiveDirectoryInteractive +user: bill.gates@microsoft.com +``` +##### ActiveDirectory Integrated (*Windows only*) +uses your machine's credentials (might be disabled by your AAD admins) +``` +authentication: ActiveDirectoryIntegrated +``` +##### Service Principal +`client_*` and `app_*` can be used interchangeably +``` +tenant_id: ActiveDirectoryIntegrated +client_id: clientid +client_secret: ActiveDirectoryIntegrated +``` +##### ActiveDirectory MSI (*to be implemented*) +``` +authentication: ActiveDirectoryMsi +``` ## Supported features diff --git a/dbt/adapters/sqlserver/connections.py b/dbt/adapters/sqlserver/connections.py index 9a3d2a2c..1ea9d780 100644 --- a/dbt/adapters/sqlserver/connections.py +++ b/dbt/adapters/sqlserver/connections.py @@ -1,11 +1,14 @@ from contextlib import contextmanager import pyodbc +import os import time +import struct import dbt.exceptions from dbt.adapters.base import Credentials from dbt.adapters.sql import SQLConnectionManager +from azure.identity import DefaultAzureCredential from dbt.logger import GLOBAL_LOGGER as logger @@ -13,6 +16,23 @@ from typing import Optional +def create_token(tenant_id, client_id, client_secret): + # bc DefaultAzureCredential will look in env variables + os.environ['AZURE_TENANT_ID'] = tenant_id + os.environ['AZURE_CLIENT_ID'] = client_id + os.environ['AZURE_CLIENT_SECRET'] = client_secret + + token = DefaultAzureCredential().get_token('https://database.windows.net//.default') + # convert to byte string interspersed with the 1-byte + # TODO decide which is cleaner? + # exptoken=b''.join([bytes({i})+bytes(1) for i in bytes(token.token, "UTF-8")]) + exptoken = bytes(1).join([bytes(i, "UTF-8") for i in token.token])+bytes(1) + # make c object with bytestring length prefix + tokenstruct = struct.pack("=i", len(exptoken)) + exptoken + + return tokenstruct + + @dataclass class SQLServerCredentials(Credentials): driver: str @@ -23,6 +43,13 @@ class SQLServerCredentials(Credentials): UID: Optional[str] = None PWD: Optional[str] = None windows_login: Optional[bool] = False + tenant_id: Optional[str] = None + client_id: Optional[str] = None + client_secret: Optional[str] = None + # "sql", "ActiveDirectoryPassword" or "ActiveDirectoryInteractive", or + # "ServicePrincipal" + authentication: Optional[str] = "sql" + encrypt: Optional[str] = "yes" _ALIASES = { 'user': 'UID' @@ -31,6 +58,9 @@ class SQLServerCredentials(Credentials): , 'password': 'PWD' , 'server': 'host' , 'trusted_connection': 'windows_login' + , 'auth': 'authentication' + , 'app_id': 'client_id' + , 'app_secret': 'client_secret' } @property @@ -40,11 +70,13 @@ def type(self): def _connection_keys(self): # return an iterator of keys to pretty-print in 'dbt debug' # raise NotImplementedError - return 'server', 'database', 'schema', 'port', 'UID', 'windows_login' + return 'server', 'database', 'schema', 'port', 'UID', \ + 'windows_login', 'authentication', 'encrypt' class SQLServerConnectionManager(SQLConnectionManager): TYPE = 'sqlserver' + TOKEN = None @contextmanager def exception_handler(self, sql): @@ -90,16 +122,55 @@ def open(cls, connection): con_str.append(f"SERVER={credentials.host},{credentials.port}") con_str.append(f"Database={credentials.database}") - if not getattr(credentials, 'windows_login', False): - con_str.append(f"UID={credentials.UID}") - con_str.append(f"PWD={credentials.PWD}") - else: + type_auth = getattr(credentials, 'authentication', 'sql') + + if 'ActiveDirectory' in type_auth: + con_str.append(f"Authentication={credentials.authentication}") + + if type_auth == "ActiveDirectoryPassword": + con_str.append(f"UID={{{credentials.UID}}}") + con_str.append(f"PWD={{{credentials.PWD}}}") + elif type_auth == "ActiveDirectoryInteractive": + con_str.append(f"UID={{{credentials.UID}}}") + elif type_auth == "ActiveDirectoryIntegrated": + # why is this necessary??? + con_str.remove("UID={None}") + elif type_auth == "ActiveDirectoryMsi": + raise ValueError("ActiveDirectoryMsi is not supported yet") + + elif type_auth == 'ServicePrincipal': + app_id = getattr(credentials, 'AppId', None) + app_secret = getattr(credentials, 'AppSecret', None) + + elif getattr(credentials, 'windows_login', False) con_str.append(f"trusted_connection=yes") + elif type_auth == 'sql': + con_str.append("Authentication=SqlPassword") + con_str.append(f"UID={{{credentials.UID}}}") + con_str.append(f"PWD={{{credentials.PWD}}}") + + if not getattr(credentials, 'encrypt', False): + con_str.append(f"Encrypt={credentials.encrypt}") con_str_concat = ';'.join(con_str) logger.debug(f'Using connection string: {con_str_concat}') - handle = pyodbc.connect(con_str_concat, autocommit=True) + if type_auth != 'ServicePrincipal': + handle = pyodbc.connect(con_str_concat, autocommit=True) + + elif type_auth == 'ServicePrincipal': + + # create token if it does not exist + if cls.TOKEN is None: + tenant_id = getattr(credentials, 'tenant_id', None) + client_id = getattr(credentials, 'client_id', None) + client_secret = getattr(credentials, 'client_secret', None) + + cls.TOKEN = create_token(tenant_id, client_id, client_secret) + + handle = pyodbc.connect(con_str_concat, + attrs_before = {1256:cls.TOKEN}, + autocommit=True) connection.state = 'open' connection.handle = handle From 76251f9d106c27a7275d1565b3394cd3dfeab408 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 4 Sep 2020 21:41:07 -0700 Subject: [PATCH 2/7] offical release is now out --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da15ef80..48277f3a 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ] }, install_requires=[ - 'dbt-core==0.18.0rc1', - 'pyodbc>=4.0.27', + 'dbt-core==0.18.0', + 'pyodbc>=4.0.27' ] ) From bc1b7dc104d0d487733c9de0422cca65c49925ff Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 4 Sep 2020 21:41:24 -0700 Subject: [PATCH 3/7] required for AD auth --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 48277f3a..dce075ee 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ }, install_requires=[ 'dbt-core==0.18.0', - 'pyodbc>=4.0.27' + 'pyodbc>=4.0.27', + 'azure-identity>=1.4.0' ] ) From 2f7024262a6f284ccfe0b4109847115f0e395910 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Mon, 14 Sep 2020 08:55:40 -0700 Subject: [PATCH 4/7] typo --- dbt/adapters/sqlserver/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/sqlserver/connections.py b/dbt/adapters/sqlserver/connections.py index 1ea9d780..c7b5e844 100644 --- a/dbt/adapters/sqlserver/connections.py +++ b/dbt/adapters/sqlserver/connections.py @@ -142,7 +142,7 @@ def open(cls, connection): app_id = getattr(credentials, 'AppId', None) app_secret = getattr(credentials, 'AppSecret', None) - elif getattr(credentials, 'windows_login', False) + elif getattr(credentials, 'windows_login', False): con_str.append(f"trusted_connection=yes") elif type_auth == 'sql': con_str.append("Authentication=SqlPassword") From 8e869bfbd93c8513537e3b307b70b8b51ce9afe9 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 17 Sep 2020 22:46:53 -0700 Subject: [PATCH 5/7] clarify auth methods --- README.md | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index bd63df89..9dceba82 100644 --- a/README.md +++ b/README.md @@ -21,15 +21,8 @@ sudo apt install unixodbc-dev ``` ## Authentication -`SqlPassword` is the default connection method, but you can also use the following [`pyodbc`-supported ActiveDirectory methods](https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#new-andor-modified-dsn-and-connection-string-keywords) to authenticate: -- Integrated (i.e. Windows Login) -- ActiveDirectory Password -- ActiveDirectory Interactive -- ActiveDirectory Integrated -- ActiveDirectory MSI (to be implemented) -- Service Principal (a.k.a. AAD Application) -#### boilerplate -this should be in every target definition + +the following is needed for every target definition for both SQL Server and Azure SQL. The sections below details how to connect to SQL Server and Azure SQL specifically. ``` type: sqlserver driver: 'ODBC Driver 17 for SQL Server' (The ODBC Driver installed on your system) @@ -37,16 +30,30 @@ server: server-host-name or ip port: 1433 schema: schemaname ``` -#### SQL Server authentication + +### standard SQL Server authentication +SQL Server credentials are supported for on-prem as well as cloud, and it is the default authentication method for `dbt-sqlsever` ``` user: username password: password ``` +### Windows Authentication (SQL Server-specific) -#### Integrated Security ``` windows_login: True ``` +alternatively +``` +trusted_connection: True +``` +### Azure SQL-specific auth +The following [`pyodbc`-supported ActiveDirectory methods](https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#new-andor-modified-dsn-and-connection-string-keywords) are available to authenticate to Azure SQL: +- ActiveDirectory Password +- ActiveDirectory Interactive +- ActiveDirectory Integrated +- Service Principal (a.k.a. AAD Application) +- ~~ActiveDirectory MSI~~ (not implemented) + #### ActiveDirectory Password Definitely not ideal, but available ``` @@ -60,7 +67,7 @@ brings up the Azure AD prompt so you can MFA if need be. authentication: ActiveDirectoryInteractive user: bill.gates@microsoft.com ``` -##### ActiveDirectory Integrated (*Windows only*) +#### ActiveDirectory Integrated (*Windows only*) uses your machine's credentials (might be disabled by your AAD admins) ``` authentication: ActiveDirectoryIntegrated @@ -72,10 +79,6 @@ tenant_id: ActiveDirectoryIntegrated client_id: clientid client_secret: ActiveDirectoryIntegrated ``` -##### ActiveDirectory MSI (*to be implemented*) -``` -authentication: ActiveDirectoryMsi -``` ## Supported features From 06aa91e2deda1d6f7ee561c5c2d515e7e2b45e31 Mon Sep 17 00:00:00 2001 From: Nandan Hegde Date: Fri, 18 Sep 2020 06:16:38 +0000 Subject: [PATCH 6/7] Windows Login used to display Auth as SQL --- dbt/adapters/sqlserver/connections.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/dbt/adapters/sqlserver/connections.py b/dbt/adapters/sqlserver/connections.py index 94c1f42a..fac7f06f 100644 --- a/dbt/adapters/sqlserver/connections.py +++ b/dbt/adapters/sqlserver/connections.py @@ -22,11 +22,12 @@ def create_token(tenant_id, client_id, client_secret): os.environ['AZURE_CLIENT_ID'] = client_id os.environ['AZURE_CLIENT_SECRET'] = client_secret - token = DefaultAzureCredential().get_token('https://database.windows.net//.default') + token = DefaultAzureCredential().get_token( + 'https://database.windows.net//.default') # convert to byte string interspersed with the 1-byte # TODO decide which is cleaner? # exptoken=b''.join([bytes({i})+bytes(1) for i in bytes(token.token, "UTF-8")]) - exptoken = bytes(1).join([bytes(i, "UTF-8") for i in token.token])+bytes(1) + exptoken = bytes(1).join([bytes(i, "UTF-8") for i in token.token])+bytes(1) # make c object with bytestring length prefix tokenstruct = struct.pack("=i", len(exptoken)) + exptoken @@ -52,15 +53,7 @@ class SQLServerCredentials(Credentials): encrypt: Optional[str] = "yes" _ALIASES = { - 'user': 'UID' - , 'username': 'UID' - , 'pass': 'PWD' - , 'password': 'PWD' - , 'server': 'host' - , 'trusted_connection': 'windows_login' - , 'auth': 'authentication' - , 'app_id': 'client_id' - , 'app_secret': 'client_secret' + 'user': 'UID', 'username': 'UID', 'pass': 'PWD', 'password': 'PWD', 'server': 'host', 'trusted_connection': 'windows_login', 'auth': 'authentication', 'app_id': 'client_id', 'app_secret': 'client_secret' } @property @@ -70,8 +63,12 @@ def type(self): def _connection_keys(self): # return an iterator of keys to pretty-print in 'dbt debug' # raise NotImplementedError + if self.windows_login is True: + self.authentication = "Windows Login" + + return 'server', 'database', 'schema', 'port', 'UID', \ - 'windows_login', 'authentication', 'encrypt' + 'authentication', 'encrypt' class SQLServerConnectionManager(SQLConnectionManager): From ae1f19120172925033ea68577ba2029370cac2e1 Mon Sep 17 00:00:00 2001 From: NandanHegde15 <67547199+NandanHegde15@users.noreply.github.com> Date: Fri, 18 Sep 2020 14:36:52 +0530 Subject: [PATCH 7/7] Update README.md Added a description of Azure SQL requiring tires greater than S2 for column store index to be created in README file --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9dceba82..bab4a106 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,9 @@ client_secret: ActiveDirectoryIntegrated ### Materializations - Table: - - Will be materialized as columns store index by default (requires SQL Server 2017 as least). To override: + - Will be materialized as columns store index by default (requires SQL Server 2017 as least). + (For Azure SQL requires Service Tier greater than S2) + To override: {{ config( as_columnstore = false,