From b01ae585c35e66b067541bdd9afb8d4658858b0f Mon Sep 17 00:00:00 2001 From: sitingren Date: Mon, 17 Jun 2024 15:06:57 +0000 Subject: [PATCH 01/22] init --- vertica_python/vertica/connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py index e52aa28e..580bcdc7 100644 --- a/vertica_python/vertica/connection.py +++ b/vertica_python/vertica/connection.py @@ -74,6 +74,7 @@ DEFAULT_REQUEST_COMPLEX_TYPES = True DEFAULT_OAUTH_ACCESS_TOKEN = '' DEFAULT_WORKLOAD = '' +DEFAULT_TLSMODE = 'prefer' try: DEFAULT_USER = getpass.getuser() except Exception as e: @@ -504,6 +505,8 @@ def _socket(self): # enable SSL ssl_options = self.options.get('ssl') self._logger.debug('SSL option is {0}'.format('enabled' if ssl_options else 'disabled')) + # If TLSmode option and SSL option are set, TLSmode option takes precedence. + tlsmode_options = self.options.get('tlsmode') if ssl_options: raw_socket = self.enable_ssl(raw_socket, ssl_options) except: From 9b6fcc653df58af397eeda1bcdf2c9dd91549a7b Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Mon, 17 Jun 2024 23:12:03 +0800 Subject: [PATCH 02/22] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c4a862b4..3fd22b1e 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ with vertica_python.connect(**conn_info) as connection: | request_complex_types | See [SQL Data conversion to Python objects](#sql-data-conversion-to-python-objects).
**_Default_**: True | | session_label | Sets a label for the connection on the server. This value appears in the client_label column of the _v_monitor.sessions_ system table.
**_Default_**: an auto-generated label with format of `vertica-python-{version}-{random_uuid}` | | ssl | See [TLS/SSL](#tlsssl).
**_Default_**: False (disabled) | +| tlsmode | See [TLS/SSL](#tlsssl).
**_Default_**: "prefer" | | unicode_error | See [UTF-8 encoding issues](#utf-8-encoding-issues).
**_Default_**: 'strict' (throw error on invalid UTF-8 results) | | use_prepared_statements | See [Passing parameters to SQL queries](#passing-parameters-to-sql-queries).
**_Default_**: False | | workload | Sets the workload name associated with this session. Valid values are workload names that already exist in a workload routing rule on the server. If a workload name that doesn't exist is entered, the server will reject it and it will be set to the default.
**_Default_**: "" | From 8c4f7e1dbdb2763efafb0c9032a89e704207e69e Mon Sep 17 00:00:00 2001 From: sitingren Date: Tue, 18 Jun 2024 04:12:10 +0000 Subject: [PATCH 03/22] add tlsmode.py --- vertica_python/vertica/tlsmode.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 vertica_python/vertica/tlsmode.py diff --git a/vertica_python/vertica/tlsmode.py b/vertica_python/vertica/tlsmode.py new file mode 100644 index 00000000..4296d965 --- /dev/null +++ b/vertica_python/vertica/tlsmode.py @@ -0,0 +1,32 @@ +# Copyright (c) 2024 Open Text. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import print_function, division, absolute_import, annotations + +from enum import Enum + +class TLSMode(Enum): + DISABLE = 'disable' + PREFER = 'prefer' + REQUIRE = 'require' + VERIFY_CA = 'verify-ca' + VERIFY_FULL = 'verify-full' + + def requests_encryption(self) -> bool: + return self != TLSMode.DISABLE + + def requires_encryption(self) -> bool: + return self not in (TLSMode.DISABLE, TLSMode.PREFER) + From b425471bf7ffcfd5a1f77fee89e61d78ff786c24 Mon Sep 17 00:00:00 2001 From: sitingren Date: Tue, 18 Jun 2024 10:55:13 +0000 Subject: [PATCH 04/22] update --- .../tests/integration_tests/test_tls.py | 23 ++++++++- .../tests/unit_tests/test_parsedsn.py | 3 +- vertica_python/vertica/connection.py | 51 ++++++++++++------- vertica_python/vertica/tlsmode.py | 15 ++++++ 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index d1e89e6a..f281d105 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -24,7 +24,11 @@ class TlsTestCase(VerticaPythonIntegrationTestCase): + SSL_STATE_SQL = 'SELECT ssl_state FROM sessions WHERE session_id=current_session()' + def tearDown(self): + if 'tlsmode' in self._conn_info: + del self._conn_info['tlsmode'] if 'ssl' in self._conn_info: del self._conn_info['ssl'] with self._connect() as conn: @@ -108,11 +112,28 @@ def test_TLSMode_disable(self): res = self._query_and_fetchone('SELECT ssl_state FROM sessions WHERE session_id=(SELECT current_session())') self.assertEqual(res[0], 'None') + def test_option_default_server_disable(self): + # TLS is disabled on the server + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'None') + + def test_option_default_server_enable(self): + # Setting certificates with TLS configuration + self._generate_and_set_certificates() + + # TLS is enabled on the server + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'Server') + def test_TLSMode_require_server_disable(self): # Requires that the server use TLS. If the TLS connection attempt fails, the client rejects the connection. self._conn_info['ssl'] = True self.assertConnectionFail(err_type=errors.SSLNotSupported, - err_msg='SSL requested but not supported by server') + err_msg='SSL requested but disabled on the server') def test_TLSMode_require(self): # Setting certificates with TLS configuration diff --git a/vertica_python/tests/unit_tests/test_parsedsn.py b/vertica_python/tests/unit_tests/test_parsedsn.py index dc78a51d..1a69845d 100644 --- a/vertica_python/tests/unit_tests/test_parsedsn.py +++ b/vertica_python/tests/unit_tests/test_parsedsn.py @@ -42,7 +42,7 @@ def test_str_arguments(self): 'session_label=vpclient&unicode_error=strict&' 'log_path=/home/admin/vClient.log&log_level=DEBUG&' 'oauth_access_token=GciOiJSUzI1NiI&' - 'workload=python_test_workload&' + 'workload=python_test_workload&tlsmode=verify-ca&' 'kerberos_service_name=krb_service&kerberos_host_name=krb_host') expected = {'database': 'db1', 'host': 'localhost', 'user': 'john', 'password': 'pwd', 'port': 5433, 'log_level': 'DEBUG', @@ -50,6 +50,7 @@ def test_str_arguments(self): 'log_path': '/home/admin/vClient.log', 'oauth_access_token': 'GciOiJSUzI1NiI', 'workload': 'python_test_workload', + 'tlsmode': 'verify-ca', 'kerberos_service_name': 'krb_service', 'kerberos_host_name': 'krb_host'} parsed = parse_dsn(dsn) diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py index 580bcdc7..ac7f9cfa 100644 --- a/vertica_python/vertica/connection.py +++ b/vertica_python/vertica/connection.py @@ -60,6 +60,7 @@ from ..vertica.messages.message import BackendMessage, FrontendMessage from ..vertica.messages.frontend_messages import CancelRequest from ..vertica.log import VerticaLogging +from ..vertica.tlsmode import TLSMode DEFAULT_HOST = 'localhost' DEFAULT_PORT = 5433 @@ -503,12 +504,27 @@ def _socket(self): raw_socket = self.balance_load(raw_socket) # enable SSL + tlsmode_options = self.options.get('tlsmode') ssl_options = self.options.get('ssl') - self._logger.debug('SSL option is {0}'.format('enabled' if ssl_options else 'disabled')) # If TLSmode option and SSL option are set, TLSmode option takes precedence. - tlsmode_options = self.options.get('tlsmode') - if ssl_options: - raw_socket = self.enable_ssl(raw_socket, ssl_options) + ssl_context = None + if tlsmode_options is not None: + tlsmode = TLSMode(tlsmode_options) + elif ssl_options is not None: + if isinstance(ssl_options, ssl.SSLContext): + ssl_context = ssl_options + tlsmode = TLSMode.REQUIRE # placeholder + elif isinstance(ssl_options, bool): + tlsmode = TLSMode.REQUIRE if ssl_options else TLSMode.DISABLE + else: + raise TypeError('The value of connection option "ssl" should be a bool or ssl.SSLContext object') + else: + tlsmode = TLSMode(DEFAULT_TLSMODE) + self._logger.debug(f'Connection TLS Mode is {tlsmode.name}') + if tlsmode.requests_encryption(): + if ssl_context is None: + ssl_context = tlsmode.get_sslcontext() + raw_socket = self.enable_ssl(raw_socket, ssl_context, force=tlsmode.requires_encryption()) except: self._logger.debug('Close the socket') raw_socket.close() @@ -571,7 +587,8 @@ def balance_load(self, raw_socket: socket.socket) -> socket.socket: def enable_ssl(self, raw_socket: socket.socket, - ssl_options: Union[ssl.SSLContext, bool]) -> ssl.SSLSocket: + ssl_context: ssl.SSLContext, + force: bool) -> Union[socket.socket, ssl.SSLSocket]: # Send SSL request and read server response self._logger.debug('=> %s', messages.SslRequest()) raw_socket.sendall(messages.SslRequest().get_message()) @@ -580,26 +597,22 @@ def enable_ssl(self, if response == b'S': self._logger.info('Enabling SSL') try: - if isinstance(ssl_options, ssl.SSLContext): - server_host = self.address_list.peek_host() - if server_host is None: # This should not happen - msg = 'Cannot get the connected server host while enabling SSL' - self._logger.error(msg) - raise errors.ConnectionError(msg) - raw_socket = ssl_options.wrap_socket(raw_socket, server_hostname=server_host) - else: - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE - raw_socket = ssl_context.wrap_socket(raw_socket) + server_host = self.address_list.peek_host() + if server_host is None: # This should not happen + msg = 'Cannot get the connected server host while enabling SSL' + self._logger.error(msg) + raise errors.ConnectionError(msg) + raw_socket = ssl_context.wrap_socket(raw_socket, server_hostname=server_host) except ssl.CertificateError as e: raise errors.ConnectionError(str(e)) except ssl.SSLError as e: raise errors.ConnectionError(str(e)) - else: - err_msg = "SSL requested but not supported by server" + elif force: + err_msg = "SSL requested but disabled on the server" self._logger.error(err_msg) raise errors.SSLNotSupported(err_msg) + else: + self._logger.info('TLS is not configured on the server. Proceeding with an unencrypted channel.') return raw_socket def establish_socket_connection(self, address_list: _AddressList) -> socket.socket: diff --git a/vertica_python/vertica/tlsmode.py b/vertica_python/vertica/tlsmode.py index 4296d965..33eae42c 100644 --- a/vertica_python/vertica/tlsmode.py +++ b/vertica_python/vertica/tlsmode.py @@ -15,6 +15,7 @@ from __future__ import print_function, division, absolute_import, annotations +import ssl from enum import Enum class TLSMode(Enum): @@ -30,3 +31,17 @@ def requests_encryption(self) -> bool: def requires_encryption(self) -> bool: return self not in (TLSMode.DISABLE, TLSMode.PREFER) + def verify_certificate(self) -> bool: + return self in (TLSMode.VERIFY_CA, TLSMode.VERIFY_FULL) + + def verify_hostname(self) -> bool: + return self == TLSMode.VERIFY_FULL + + def get_sslcontext(self, cafile=None) -> ssl.SSLContext: + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.check_hostname = self.verify_hostname() + ssl_context.verify_mode = ssl.CERT_REQUIRED if self.verify_certificate() else ssl.CERT_NONE + if cafile: + ssl_context.load_verify_locations(cafile=cafile) + return ssl_context + From b6f798d3b5b1359efb63c4398aac7cb6142c7fe2 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Tue, 18 Jun 2024 22:02:55 +0800 Subject: [PATCH 05/22] update test --- vertica_python/tests/integration_tests/test_tls.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index f281d105..5b5c35d8 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -109,7 +109,7 @@ def test_TLSMode_disable(self): self._conn_info['ssl'] = False with self._connect() as conn: cur = conn.cursor() - res = self._query_and_fetchone('SELECT ssl_state FROM sessions WHERE session_id=(SELECT current_session())') + res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'None') def test_option_default_server_disable(self): @@ -143,7 +143,7 @@ def test_TLSMode_require(self): self._conn_info['ssl'] = True with self._connect() as conn: cur = conn.cursor() - res = self._query_and_fetchone('SELECT ssl_state FROM sessions WHERE session_id=(SELECT current_session())') + res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') # Option 2 @@ -153,7 +153,7 @@ def test_TLSMode_require(self): self._conn_info['ssl'] = ssl_context with self._connect() as conn: cur = conn.cursor() - res = self._query_and_fetchone('SELECT ssl_state FROM sessions WHERE session_id=(SELECT current_session())') + res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') def test_TLSMode_verify_ca(self): @@ -168,7 +168,7 @@ def test_TLSMode_verify_ca(self): with self._connect() as conn: cur = conn.cursor() - res = self._query_and_fetchone('SELECT ssl_state FROM sessions WHERE session_id=(SELECT current_session())') + res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') def test_TLSMode_verify_full(self): @@ -183,7 +183,7 @@ def test_TLSMode_verify_full(self): self._conn_info['ssl'] = ssl_context with self._connect() as conn: cur = conn.cursor() - res = self._query_and_fetchone('SELECT ssl_state FROM sessions WHERE session_id=(SELECT current_session())') + res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') def test_mutual_TLS(self): @@ -199,7 +199,7 @@ def test_mutual_TLS(self): self._conn_info['ssl'] = ssl_context with self._connect() as conn: cur = conn.cursor() - res = self._query_and_fetchone('SELECT ssl_state FROM sessions WHERE session_id=(SELECT current_session())') + res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Mutual') From 71579dfd14520c8be103031924533a0bc095c9f8 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Tue, 18 Jun 2024 22:46:52 +0800 Subject: [PATCH 06/22] update prefer tls --- vertica_python/vertica/connection.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py index ac7f9cfa..5ffeb000 100644 --- a/vertica_python/vertica/connection.py +++ b/vertica_python/vertica/connection.py @@ -594,6 +594,7 @@ def enable_ssl(self, raw_socket.sendall(messages.SslRequest().get_message()) response = raw_socket.recv(1) self._logger.debug('<= SslResponse: %s', response) + exception = None if response == b'S': self._logger.info('Enabling SSL') try: @@ -604,15 +605,19 @@ def enable_ssl(self, raise errors.ConnectionError(msg) raw_socket = ssl_context.wrap_socket(raw_socket, server_hostname=server_host) except ssl.CertificateError as e: - raise errors.ConnectionError(str(e)) + exception = errors.ConnectionError(str(e)) except ssl.SSLError as e: - raise errors.ConnectionError(str(e)) - elif force: - err_msg = "SSL requested but disabled on the server" - self._logger.error(err_msg) - raise errors.SSLNotSupported(err_msg) + exception = errors.ConnectionError(str(e)) else: - self._logger.info('TLS is not configured on the server. Proceeding with an unencrypted channel.') + err_msg = "SSL requested but disabled on the server" + exception = errors.SSLNotSupported(err_msg) + + if exception is not None: + self._logger.error(str(e)) + if force: + raise exception + else: + self._logger.warning('Cannot enable TLS. Proceeding with an unencrypted channel.') return raw_socket def establish_socket_connection(self, address_list: _AddressList) -> socket.socket: From 90ebef8dfd884d75e49600dcf60139878d8085fb Mon Sep 17 00:00:00 2001 From: sitingren Date: Wed, 19 Jun 2024 05:05:37 +0000 Subject: [PATCH 07/22] fix test --- .../tests/integration_tests/test_tls.py | 11 +++++--- vertica_python/vertica/connection.py | 26 +++++++++---------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index 5b5c35d8..c3e0d39b 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -27,10 +27,9 @@ class TlsTestCase(VerticaPythonIntegrationTestCase): SSL_STATE_SQL = 'SELECT ssl_state FROM sessions WHERE session_id=current_session()' def tearDown(self): - if 'tlsmode' in self._conn_info: - del self._conn_info['tlsmode'] - if 'ssl' in self._conn_info: - del self._conn_info['ssl'] + # Use a non-TLS connection here so cleanup can happen + # even if mutual mode is configured + self._conn_info['tlsmode'] = 'disable' with self._connect() as conn: cur = conn.cursor() cur.execute("ALTER TLS CONFIGURATION server CERTIFICATE NULL TLSMODE 'DISABLE'") @@ -42,6 +41,10 @@ def tearDown(self): cur.execute("DROP KEY IF EXISTS vp_client_key CASCADE") cur.execute("DROP KEY IF EXISTS vp_server_key CASCADE") cur.execute("DROP KEY IF EXISTS vp_CA_key CASCADE") + if 'tlsmode' in self._conn_info: + del self._conn_info['tlsmode'] + if 'ssl' in self._conn_info: + del self._conn_info['ssl'] super(TlsTestCase, self).tearDown() def _generate_and_set_certificates(self, mutual_mode=False): diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py index 5ffeb000..107ca0c7 100644 --- a/vertica_python/vertica/connection.py +++ b/vertica_python/vertica/connection.py @@ -594,30 +594,28 @@ def enable_ssl(self, raw_socket.sendall(messages.SslRequest().get_message()) response = raw_socket.recv(1) self._logger.debug('<= SslResponse: %s', response) - exception = None if response == b'S': - self._logger.info('Enabling SSL') + self._logger.info('Enabling TLS') try: server_host = self.address_list.peek_host() if server_host is None: # This should not happen - msg = 'Cannot get the connected server host while enabling SSL' + msg = 'Cannot get the connected server host while enabling TLS' self._logger.error(msg) raise errors.ConnectionError(msg) raw_socket = ssl_context.wrap_socket(raw_socket, server_hostname=server_host) except ssl.CertificateError as e: - exception = errors.ConnectionError(str(e)) + raise errors.ConnectionError(str(e)) except ssl.SSLError as e: - exception = errors.ConnectionError(str(e)) - else: + raise errors.ConnectionError(str(e)) + elif force: err_msg = "SSL requested but disabled on the server" - exception = errors.SSLNotSupported(err_msg) - - if exception is not None: - self._logger.error(str(e)) - if force: - raise exception - else: - self._logger.warning('Cannot enable TLS. Proceeding with an unencrypted channel.') + self._logger.error(err_msg) + raise errors.SSLNotSupported(err_msg) + else: + msg = 'TLS is not configured on the server. Proceeding with an unencrypted channel.' + hint = "\nHINT: Set connection option 'tlsmode' to 'disable' to explicitly create a non-TLS connection." + warnings.warn(msg + hint) + self._logger.warning(msg) return raw_socket def establish_socket_connection(self, address_list: _AddressList) -> socket.socket: From 08a4b33912cb590af8cc30fb08aaca6dd51206d1 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Wed, 19 Jun 2024 19:49:27 +0800 Subject: [PATCH 08/22] update test --- .../tests/integration_tests/test_tls.py | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index c3e0d39b..e9429c92 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -108,25 +108,48 @@ def _generate_and_set_certificates(self, mutual_mode=False): return vp_CA_cert + def test_option_default_server_disable(self): + # TLS is disabled on the server + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'None') + + def test_option_default_server_enable(self): + # Setting certificates with TLS configuration + self._generate_and_set_certificates() + + # TLS is enabled on the server + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'Server') + def test_TLSMode_disable(self): - self._conn_info['ssl'] = False + self._conn_info['tlsmode'] = 'disable' with self._connect() as conn: cur = conn.cursor() res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'None') - def test_option_default_server_disable(self): - # TLS is disabled on the server + def test_ssl_false(self): + self._conn_info['ssl'] = False with self._connect() as conn: cur = conn.cursor() res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'None') - def test_option_default_server_enable(self): + def test_ssl_true_server_disable(self): + # Requires that the server use TLS. If the TLS connection attempt fails, the client rejects the connection. + self._conn_info['ssl'] = True + self.assertConnectionFail(err_type=errors.SSLNotSupported, + err_msg='SSL requested but disabled on the server') + + def test_ssl_true_server_enable(self): # Setting certificates with TLS configuration self._generate_and_set_certificates() - # TLS is enabled on the server + self._conn_info['ssl'] = True with self._connect() as conn: cur = conn.cursor() res = self._query_and_fetchone(self.SSL_STATE_SQL) @@ -134,22 +157,24 @@ def test_option_default_server_enable(self): def test_TLSMode_require_server_disable(self): # Requires that the server use TLS. If the TLS connection attempt fails, the client rejects the connection. - self._conn_info['ssl'] = True + self._conn_info['tlsmode'] = 'require' self.assertConnectionFail(err_type=errors.SSLNotSupported, err_msg='SSL requested but disabled on the server') - def test_TLSMode_require(self): + def test_TLSMode_require_server_enable(self): # Setting certificates with TLS configuration self._generate_and_set_certificates() - # Option 1 - self._conn_info['ssl'] = True + self._conn_info['tlsmode'] = 'require' with self._connect() as conn: cur = conn.cursor() res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') - # Option 2 + def test_sslcontext_require(self): + # Setting certificates with TLS configuration + self._generate_and_set_certificates() + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE From 662ffc427361b881b4d7e7f87e576ce5465e09b4 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Wed, 19 Jun 2024 22:20:27 +0800 Subject: [PATCH 09/22] update test --- .../tests/integration_tests/test_tls.py | 55 +++++++++++++++++-- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index e9429c92..85cb6019 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -91,7 +91,7 @@ def _generate_and_set_certificates(self, mutual_mode=False): # This CA certificate is used to verify client certificates cur.execute('ALTER TLS CONFIGURATION server CERTIFICATE vp_server_cert ADD CA CERTIFICATES vp_CA_cert') # Enable TLS. Connection succeeds if Vertica verifies that the client certificate is from a trusted CA. - # If the client does not present a client certificate, the connection uses plaintext. + # If the client does not present a client certificate, the connection is rejected. cur.execute("ALTER TLS CONFIGURATION server TLSMODE 'VERIFY_CA'") else: @@ -107,6 +107,9 @@ def _generate_and_set_certificates(self, mutual_mode=False): return vp_CA_cert + ###################################################### + #### Test 'ssl' and 'tlsmode' options are not set #### + ###################################################### def test_option_default_server_disable(self): # TLS is disabled on the server @@ -125,13 +128,22 @@ def test_option_default_server_enable(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') - def test_TLSMode_disable(self): + ####################################################### + #### Test 'ssl' and 'tlsmode' options are both set #### + ####################################################### + + def test_tlsmode_over_ssl(self): self._conn_info['tlsmode'] = 'disable' + self._conn_info['ssl'] = True with self._connect() as conn: cur = conn.cursor() res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'None') + ############################################### + #### Test 'ssl' option with boolean values #### + ############################################### + def test_ssl_false(self): self._conn_info['ssl'] = False with self._connect() as conn: @@ -155,6 +167,35 @@ def test_ssl_true_server_enable(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') + ############################### + #### Test 'tlsmode' option #### + ############################### + + def test_TLSMode_disable(self): + self._conn_info['tlsmode'] = 'disable' + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'None') + + def test_TLSMode_prefer_server_disable(self): + # TLS is disabled on the server + self._conn_info['tlsmode'] = 'prefer' + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'None') + + def test_TLSMode_prefer_server_enable(self): + # Setting certificates with TLS configuration + self._generate_and_set_certificates() + + self._conn_info['tlsmode'] = 'prefer' + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'Server') + def test_TLSMode_require_server_disable(self): # Requires that the server use TLS. If the TLS connection attempt fails, the client rejects the connection. self._conn_info['tlsmode'] = 'require' @@ -171,6 +212,10 @@ def test_TLSMode_require_server_enable(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') + ###################################################### + #### Test 'ssl' option with ssl.SSLContext object #### + ###################################################### + def test_sslcontext_require(self): # Setting certificates with TLS configuration self._generate_and_set_certificates() @@ -184,7 +229,7 @@ def test_sslcontext_require(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') - def test_TLSMode_verify_ca(self): + def test_sslcontext_verify_ca(self): # Setting certificates with TLS configuration CA_cert = self._generate_and_set_certificates() @@ -199,7 +244,7 @@ def test_TLSMode_verify_ca(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') - def test_TLSMode_verify_full(self): + def test_sslcontext_verify_full(self): # Setting certificates with TLS configuration CA_cert = self._generate_and_set_certificates() @@ -214,7 +259,7 @@ def test_TLSMode_verify_full(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') - def test_mutual_TLS(self): + def test_sslcontext_mutual_TLS(self): # Setting certificates with TLS configuration CA_cert = self._generate_and_set_certificates(mutual_mode=True) From 8fba157cd04c7af47d46a85e923b112960ac07ee Mon Sep 17 00:00:00 2001 From: sitingren Date: Fri, 21 Jun 2024 14:36:10 +0000 Subject: [PATCH 10/22] update --- vertica_python/tests/integration_tests/test_tls.py | 11 +++++++++++ vertica_python/tests/unit_tests/test_parsedsn.py | 2 ++ vertica_python/vertica/connection.py | 5 +++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index 85cb6019..139de880 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -212,6 +212,17 @@ def test_TLSMode_require_server_enable(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') + def test_TLSMode_verify_ca(self): + # Setting certificates with TLS configuration + CA_cert = self._generate_and_set_certificates() + + self._conn_info['tlsmode'] = 'verify_ca' + self._conn_info['tls_cafile'] = 'p' + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'Server') + ###################################################### #### Test 'ssl' option with ssl.SSLContext object #### ###################################################### diff --git a/vertica_python/tests/unit_tests/test_parsedsn.py b/vertica_python/tests/unit_tests/test_parsedsn.py index 1a69845d..b87aab4f 100644 --- a/vertica_python/tests/unit_tests/test_parsedsn.py +++ b/vertica_python/tests/unit_tests/test_parsedsn.py @@ -43,6 +43,7 @@ def test_str_arguments(self): 'log_path=/home/admin/vClient.log&log_level=DEBUG&' 'oauth_access_token=GciOiJSUzI1NiI&' 'workload=python_test_workload&tlsmode=verify-ca&' + 'tls_cafile=tls/ca_cert.pem&' 'kerberos_service_name=krb_service&kerberos_host_name=krb_host') expected = {'database': 'db1', 'host': 'localhost', 'user': 'john', 'password': 'pwd', 'port': 5433, 'log_level': 'DEBUG', @@ -51,6 +52,7 @@ def test_str_arguments(self): 'oauth_access_token': 'GciOiJSUzI1NiI', 'workload': 'python_test_workload', 'tlsmode': 'verify-ca', + 'tls_cafile': 'tls/ca_cert.pem', 'kerberos_service_name': 'krb_service', 'kerberos_host_name': 'krb_host'} parsed = parse_dsn(dsn) diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py index 107ca0c7..2a8318a1 100644 --- a/vertica_python/vertica/connection.py +++ b/vertica_python/vertica/connection.py @@ -503,7 +503,7 @@ def _socket(self): if load_balance_options: raw_socket = self.balance_load(raw_socket) - # enable SSL + # enable TLS tlsmode_options = self.options.get('tlsmode') ssl_options = self.options.get('ssl') # If TLSmode option and SSL option are set, TLSmode option takes precedence. @@ -523,7 +523,8 @@ def _socket(self): self._logger.debug(f'Connection TLS Mode is {tlsmode.name}') if tlsmode.requests_encryption(): if ssl_context is None: - ssl_context = tlsmode.get_sslcontext() + cafile = self.options.get('tls_cafile') + ssl_context = tlsmode.get_sslcontext(cafile=cafile) raw_socket = self.enable_ssl(raw_socket, ssl_context, force=tlsmode.requires_encryption()) except: self._logger.debug('Close the socket') From 8597d70f8f6f1f8e11768690a597f5512945a6c9 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Fri, 21 Jun 2024 22:44:23 +0800 Subject: [PATCH 11/22] test_TLSMode_verify_ca --- vertica_python/tests/integration_tests/test_tls.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index 139de880..1a5bedf9 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -39,6 +39,7 @@ def tearDown(self): if hasattr(self, 'client_key'): os.remove(self.client_key.name) cur.execute("DROP KEY IF EXISTS vp_client_key CASCADE") + os.remove(self.CA_cert.name) cur.execute("DROP KEY IF EXISTS vp_server_key CASCADE") cur.execute("DROP KEY IF EXISTS vp_CA_key CASCADE") if 'tlsmode' in self._conn_info: @@ -59,6 +60,8 @@ def _generate_and_set_certificates(self, mutual_mode=False): "VALID FOR 3650 EXTENSIONS 'nsComment' = 'Self-signed root CA cert' KEY vp_CA_key") cur.execute("SELECT certificate_text FROM CERTIFICATES WHERE name='vp_CA_cert'") vp_CA_cert = cur.fetchone()[0] + with NamedTemporaryFile(delete=False) as self.CA_cert: + self.CA_cert.write(vp_CA_cert.encode()) # Generate a server private key cur.execute("CREATE KEY vp_server_key TYPE 'RSA' LENGTH 4096") @@ -217,7 +220,7 @@ def test_TLSMode_verify_ca(self): CA_cert = self._generate_and_set_certificates() self._conn_info['tlsmode'] = 'verify_ca' - self._conn_info['tls_cafile'] = 'p' + self._conn_info['tls_cafile'] = self.CA_cert.name with self._connect() as conn: cur = conn.cursor() res = self._query_and_fetchone(self.SSL_STATE_SQL) From cf46418fca2622da9841c70df52c90f8894f804a Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Fri, 21 Jun 2024 22:50:39 +0800 Subject: [PATCH 12/22] test_TLSMode_verify_ca --- vertica_python/tests/integration_tests/test_tls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index 1a5bedf9..d62406e0 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -39,7 +39,8 @@ def tearDown(self): if hasattr(self, 'client_key'): os.remove(self.client_key.name) cur.execute("DROP KEY IF EXISTS vp_client_key CASCADE") - os.remove(self.CA_cert.name) + if hasattr(self, 'CA_cert'): + os.remove(self.CA_cert.name) cur.execute("DROP KEY IF EXISTS vp_server_key CASCADE") cur.execute("DROP KEY IF EXISTS vp_CA_key CASCADE") if 'tlsmode' in self._conn_info: From 072b8080821e7b1145e427726638780cc1aa4589 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Fri, 21 Jun 2024 23:12:22 +0800 Subject: [PATCH 13/22] fix typo --- README.md | 1 + vertica_python/tests/integration_tests/test_tls.py | 2 +- vertica_python/vertica/tlsmode.py | 9 ++++++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3fd22b1e..4f43ee10 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ with vertica_python.connect(**conn_info) as connection: | session_label | Sets a label for the connection on the server. This value appears in the client_label column of the _v_monitor.sessions_ system table.
**_Default_**: an auto-generated label with format of `vertica-python-{version}-{random_uuid}` | | ssl | See [TLS/SSL](#tlsssl).
**_Default_**: False (disabled) | | tlsmode | See [TLS/SSL](#tlsssl).
**_Default_**: "prefer" | +| tls_cafile | The name of a file containing SSL certificate authority (CA) certificate(s).
See [TLS/SSL](#tlsssl). | | unicode_error | See [UTF-8 encoding issues](#utf-8-encoding-issues).
**_Default_**: 'strict' (throw error on invalid UTF-8 results) | | use_prepared_statements | See [Passing parameters to SQL queries](#passing-parameters-to-sql-queries).
**_Default_**: False | | workload | Sets the workload name associated with this session. Valid values are workload names that already exist in a workload routing rule on the server. If a workload name that doesn't exist is entered, the server will reject it and it will be set to the default.
**_Default_**: "" | diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index d62406e0..19600e8a 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -220,7 +220,7 @@ def test_TLSMode_verify_ca(self): # Setting certificates with TLS configuration CA_cert = self._generate_and_set_certificates() - self._conn_info['tlsmode'] = 'verify_ca' + self._conn_info['tlsmode'] = 'verify-ca' self._conn_info['tls_cafile'] = self.CA_cert.name with self._connect() as conn: cur = conn.cursor() diff --git a/vertica_python/vertica/tlsmode.py b/vertica_python/vertica/tlsmode.py index 33eae42c..dd2f115c 100644 --- a/vertica_python/vertica/tlsmode.py +++ b/vertica_python/vertica/tlsmode.py @@ -40,8 +40,11 @@ def verify_hostname(self) -> bool: def get_sslcontext(self, cafile=None) -> ssl.SSLContext: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.check_hostname = self.verify_hostname() - ssl_context.verify_mode = ssl.CERT_REQUIRED if self.verify_certificate() else ssl.CERT_NONE - if cafile: - ssl_context.load_verify_locations(cafile=cafile) + if self.verify_certificate(): + ssl_context.verify_mode = ssl.CERT_REQUIRED + if cafile: + ssl_context.load_verify_locations(cafile=cafile) + else: + ssl_context.verify_mode = ssl.CERT_NONE return ssl_context From ba67fea0b7b07ad705b1ce91b05880b95cf4176e Mon Sep 17 00:00:00 2001 From: sitingren Date: Mon, 24 Jun 2024 08:40:29 +0000 Subject: [PATCH 14/22] client cert and key --- vertica_python/vertica/connection.py | 4 +++- vertica_python/vertica/tlsmode.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py index 2a8318a1..15bc9446 100644 --- a/vertica_python/vertica/connection.py +++ b/vertica_python/vertica/connection.py @@ -524,7 +524,9 @@ def _socket(self): if tlsmode.requests_encryption(): if ssl_context is None: cafile = self.options.get('tls_cafile') - ssl_context = tlsmode.get_sslcontext(cafile=cafile) + certfile = self.options.get('tls_certfile') + keyfile = self.options.get('tls_keyfile') + ssl_context = tlsmode.get_sslcontext(cafile, certfile, keyfile) raw_socket = self.enable_ssl(raw_socket, ssl_context, force=tlsmode.requires_encryption()) except: self._logger.debug('Close the socket') diff --git a/vertica_python/vertica/tlsmode.py b/vertica_python/vertica/tlsmode.py index dd2f115c..d665aef2 100644 --- a/vertica_python/vertica/tlsmode.py +++ b/vertica_python/vertica/tlsmode.py @@ -17,6 +17,10 @@ import ssl from enum import Enum +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from typing import Optional + class TLSMode(Enum): DISABLE = 'disable' @@ -37,7 +41,9 @@ def verify_certificate(self) -> bool: def verify_hostname(self) -> bool: return self == TLSMode.VERIFY_FULL - def get_sslcontext(self, cafile=None) -> ssl.SSLContext: + def get_sslcontext(self, cafile: Optional[str] = None, + certfile: Optional[str] = None, + keyfile: Optional[str] = None) -> ssl.SSLContext: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.check_hostname = self.verify_hostname() if self.verify_certificate(): @@ -46,5 +52,8 @@ def get_sslcontext(self, cafile=None) -> ssl.SSLContext: ssl_context.load_verify_locations(cafile=cafile) else: ssl_context.verify_mode = ssl.CERT_NONE + # mutual mode + if certfile or keyfile: + ssl_context.load_cert_chain(certfile=certfile, keyfile=keyfile) return ssl_context From 0cbeabc41eacf6dbed291e754358255df1d773a6 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Mon, 24 Jun 2024 16:57:23 +0800 Subject: [PATCH 15/22] add test --- .../tests/integration_tests/test_tls.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index 19600e8a..2ca667ba 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -227,6 +227,30 @@ def test_TLSMode_verify_ca(self): res = self._query_and_fetchone(self.SSL_STATE_SQL) self.assertEqual(res[0], 'Server') + def test_TLSMode_verify_full(self): + # Setting certificates with TLS configuration + CA_cert = self._generate_and_set_certificates() + + self._conn_info['tlsmode'] = 'verify-full' + self._conn_info['tls_cafile'] = self.CA_cert.name + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'Server') + + def test_TLSMode_mutual_TLS(self): + # Setting certificates with TLS configuration + CA_cert = self._generate_and_set_certificates(mutual_mode=True) + + self._conn_info['tlsmode'] = 'verify-full' + self._conn_info['tls_cafile'] = self.CA_cert.name # CA certificate used to verify server certificate + self._conn_info['tls_certfile'] = self.client_cert.name # client certificate + self._conn_info['tls_keyfile'] = self.client_key.name # private key used for the client certificate + with self._connect() as conn: + cur = conn.cursor() + res = self._query_and_fetchone(self.SSL_STATE_SQL) + self.assertEqual(res[0], 'Mutual') + ###################################################### #### Test 'ssl' option with ssl.SSLContext object #### ###################################################### From d8616ca7f787df3f11a0b7398ada322781fa8df8 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Mon, 24 Jun 2024 17:09:42 +0800 Subject: [PATCH 16/22] update test --- vertica_python/tests/integration_tests/test_tls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vertica_python/tests/integration_tests/test_tls.py b/vertica_python/tests/integration_tests/test_tls.py index 2ca667ba..62e120b0 100644 --- a/vertica_python/tests/integration_tests/test_tls.py +++ b/vertica_python/tests/integration_tests/test_tls.py @@ -43,10 +43,10 @@ def tearDown(self): os.remove(self.CA_cert.name) cur.execute("DROP KEY IF EXISTS vp_server_key CASCADE") cur.execute("DROP KEY IF EXISTS vp_CA_key CASCADE") - if 'tlsmode' in self._conn_info: - del self._conn_info['tlsmode'] - if 'ssl' in self._conn_info: - del self._conn_info['ssl'] + + for key in ('tlsmode', 'ssl', 'tls_cafile', 'tls_certfile', 'tls_keyfile'): + if key in self._conn_info: + del self._conn_info[key] super(TlsTestCase, self).tearDown() def _generate_and_set_certificates(self, mutual_mode=False): From 4fd13dc6fa3d5661fec24f2e9c270260d7570b4b Mon Sep 17 00:00:00 2001 From: sitingren Date: Mon, 24 Jun 2024 13:03:21 +0000 Subject: [PATCH 17/22] add warning --- vertica_python/vertica/tlsmode.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vertica_python/vertica/tlsmode.py b/vertica_python/vertica/tlsmode.py index d665aef2..66dbae81 100644 --- a/vertica_python/vertica/tlsmode.py +++ b/vertica_python/vertica/tlsmode.py @@ -16,6 +16,7 @@ from __future__ import print_function, division, absolute_import, annotations import ssl +import warnings from enum import Enum from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -50,10 +51,14 @@ def get_sslcontext(self, cafile: Optional[str] = None, ssl_context.verify_mode = ssl.CERT_REQUIRED if cafile: ssl_context.load_verify_locations(cafile=cafile) + # mutual mode + if certfile or keyfile: + ssl_context.load_cert_chain(certfile=certfile, keyfile=keyfile) else: ssl_context.verify_mode = ssl.CERT_NONE - # mutual mode - if certfile or keyfile: - ssl_context.load_cert_chain(certfile=certfile, keyfile=keyfile) + if cafile or certfile or keyfile: + ignore_cert_msg = ("Ignore TLS certificate files and skip certificates" + " validation as tlsmode is not 'verify-ca' or 'verify-full'.") + warnings.warn(ignore_cert_msg) return ssl_context From 6f8479dec8c80aaccc71cc5c9defe5e50fcf1bfb Mon Sep 17 00:00:00 2001 From: sitingren Date: Mon, 24 Jun 2024 13:16:21 +0000 Subject: [PATCH 18/22] update readme --- README.md | 1 + vertica_python/vertica/tlsmode.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f43ee10..e7eab989 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ with vertica_python.connect(**conn_info) as connection: | ssl | See [TLS/SSL](#tlsssl).
**_Default_**: False (disabled) | | tlsmode | See [TLS/SSL](#tlsssl).
**_Default_**: "prefer" | | tls_cafile | The name of a file containing SSL certificate authority (CA) certificate(s).
See [TLS/SSL](#tlsssl). | +| tls_certfile | The name of a file containing client's certificate(s).
See [TLS/SSL](#tlsssl). | | unicode_error | See [UTF-8 encoding issues](#utf-8-encoding-issues).
**_Default_**: 'strict' (throw error on invalid UTF-8 results) | | use_prepared_statements | See [Passing parameters to SQL queries](#passing-parameters-to-sql-queries).
**_Default_**: False | | workload | Sets the workload name associated with this session. Valid values are workload names that already exist in a workload routing rule on the server. If a workload name that doesn't exist is entered, the server will reject it and it will be set to the default.
**_Default_**: "" | diff --git a/vertica_python/vertica/tlsmode.py b/vertica_python/vertica/tlsmode.py index 66dbae81..29186bce 100644 --- a/vertica_python/vertica/tlsmode.py +++ b/vertica_python/vertica/tlsmode.py @@ -58,7 +58,8 @@ def get_sslcontext(self, cafile: Optional[str] = None, ssl_context.verify_mode = ssl.CERT_NONE if cafile or certfile or keyfile: ignore_cert_msg = ("Ignore TLS certificate files and skip certificates" - " validation as tlsmode is not 'verify-ca' or 'verify-full'.") + f" validation as tlsmode is not '{TLSMode.VERIFY_CA.value}'" + f" or '{TLSMode.VERIFY_FULL.value}'.") warnings.warn(ignore_cert_msg) return ssl_context From c07c4bd26c11304a1c3b431e865e6e727f233ad4 Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Mon, 24 Jun 2024 23:19:21 +0800 Subject: [PATCH 19/22] Update readme --- README.md | 62 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e7eab989..9c46bc65 100644 --- a/README.md +++ b/README.md @@ -106,10 +106,11 @@ with vertica_python.connect(**conn_info) as connection: | oauth_access_token | See [OAuth Authentication](#oauth-authentication).
**_Default_**: "" | | request_complex_types | See [SQL Data conversion to Python objects](#sql-data-conversion-to-python-objects).
**_Default_**: True | | session_label | Sets a label for the connection on the server. This value appears in the client_label column of the _v_monitor.sessions_ system table.
**_Default_**: an auto-generated label with format of `vertica-python-{version}-{random_uuid}` | -| ssl | See [TLS/SSL](#tlsssl).
**_Default_**: False (disabled) | -| tlsmode | See [TLS/SSL](#tlsssl).
**_Default_**: "prefer" | +| ssl | See [TLS/SSL](#tlsssl).
**_Default_**: None (tlsmode="prefer") | +| tlsmode | Controls whether the connection to the server uses TLS encryption.
See [TLS/SSL](#tlsssl).
**_Default_**: "prefer" | | tls_cafile | The name of a file containing SSL certificate authority (CA) certificate(s).
See [TLS/SSL](#tlsssl). | | tls_certfile | The name of a file containing client's certificate(s).
See [TLS/SSL](#tlsssl). | +| tls_keyfile | The name of a file containing client's private key.
See [TLS/SSL](#tlsssl). | | unicode_error | See [UTF-8 encoding issues](#utf-8-encoding-issues).
**_Default_**: 'strict' (throw error on invalid UTF-8 results) | | use_prepared_statements | See [Passing parameters to SQL queries](#passing-parameters-to-sql-queries).
**_Default_**: False | | workload | Sets the workload name associated with this session. Valid values are workload names that already exist in a workload routing rule on the server. If a workload name that doesn't exist is entered, the server will reject it and it will be set to the default.
**_Default_**: "" | @@ -144,22 +145,61 @@ with vertica_python.connect(dsn=connection_str, **additional_info) as conn: ``` #### TLS/SSL -You can pass `True` to `ssl` to enable TLS/SSL connection (equivalent to TLSMode=require). + +There are two options to control client-server TLS: `tlsmode` and `ssl`. If both are set, `tlsmode` takes precedence. + +`ssl` can be a bool or a `ssl.SSLContext` object. Here is the value mapping between `ssl` (exclude `ssl.SSLContext`) and `tlsmode`: + +| `tlsmode` | `ssl` | Description | +| ------------- | ------------- | ---| +| 'disable' | False | only try a non-TLS connection. | +| 'prefer' | (not set) | (Default) first try a TLS connection; if TLS is disabled on the server, then fallback to a non-TLS connection.
Note: If TLS is enabled on the server and TLS connection fails, the client rejects the connection. | +| 'require' | True | connects using TLS without verifying certificates. If the TLS connection attempt fails, the client rejects the connection. | +| 'verify-ca' || connects using TLS and confirms that the server certificate has been signed by the certificate authority. | +| 'verify-full' ||connects using TLS, confirms that the server certificate has been signed by the certificate authority, and verifies that the host name matches the name provided in the server certificate. | + +When `tlsmode` is 'verify-ca' or 'verify-full', these options take certificate/key files: `tls_cafile`, `tls_certfile` and `tls_keyfile`. Otherwise, these options are ignored. + +`tlsmode` example: +```python +# [TLSMode: require] +import vertica_python +conn_info = {'host': '127.0.0.1', + 'user': 'some_user', + 'database': 'a_database', + 'tlsmode': 'require'} +connection = vertica_python.connect(**conn_info) +``` +```python +# [TLSMode: verify-ca] +import vertica_python +conn_info = {'host': '127.0.0.1', + 'user': 'some_user', + 'database': 'a_database', + 'tlsmode': 'verify-ca', + 'tls_cafile': '/path/to/ca_file.pem' # CA certificate used to verify server certificate + } +connection = vertica_python.connect(**conn_info) +``` ```python +# [TLSMode: verify-full] + Mutual Mode import vertica_python -# [TLSMode: require] conn_info = {'host': '127.0.0.1', - 'port': 5433, 'user': 'some_user', - 'password': 'some_password', 'database': 'a_database', - 'ssl': True} + 'tlsmode': 'verify-full', + 'tls_cafile' = '/path/to/ca_file.pem' # CA certificate used to verify server certificate + 'tls_certfile' = '/path/to/client.pem', # (for mutual mode) client certificate + 'tls_keyfile' = '/path/to/client.key' # (for mutual mode) client private key + } connection = vertica_python.connect(**conn_info) ``` -You can pass an `ssl.SSLContext` to `ssl` to customize the SSL connection options. Server mode TLS examples: +You can pass an `ssl.SSLContext` object to `ssl` to customize the underlying SSL connection options. See more on SSL options [here](https://docs.python.org/3/library/ssl.html). + +Server mode TLS examples: ```python import vertica_python @@ -174,7 +214,6 @@ ssl_context.verify_mode = ssl.CERT_NONE conn_info = {'host': '127.0.0.1', 'port': 5433, 'user': 'some_user', - 'password': 'some_password', 'database': 'a_database', 'ssl': ssl_context} connection = vertica_python.connect(**conn_info) @@ -190,7 +229,6 @@ ssl_context.load_verify_locations(cafile='/path/to/ca_file.pem') # CA certificat conn_info = {'host': '127.0.0.1', 'port': 5433, 'user': 'some_user', - 'password': 'some_password', 'database': 'a_database', 'ssl': ssl_context} connection = vertica_python.connect(**conn_info) @@ -207,7 +245,6 @@ ssl_context.load_verify_locations(cafile='/path/to/ca_file.pem') # CA certificat conn_info = {'host': '127.0.0.1', 'port': 5433, 'user': 'some_user', - 'password': 'some_password', 'database': 'a_database', 'ssl': ssl_context} connection = vertica_python.connect(**conn_info) @@ -233,13 +270,12 @@ ssl_context.load_cert_chain(certfile='/path/to/client.pem', keyfile='/path/to/cl conn_info = {'host': '127.0.0.1', 'port': 5433, 'user': 'some_user', - 'password': 'some_password', 'database': 'a_database', 'ssl': ssl_context} connection = vertica_python.connect(**conn_info) ``` -See more on SSL options [here](https://docs.python.org/3/library/ssl.html). + #### Kerberos Authentication In order to use Kerberos authentication, install [dependencies](#using-kerberos-authentication) first, and it is the user's responsibility to ensure that an Ticket-Granting Ticket (TGT) is available and valid. Whether a TGT is available can be easily determined by running the `klist` command. If no TGT is available, then it first must be obtained by running the `kinit` command or by logging in. You can pass in optional arguments to customize the authentication. The arguments are `kerberos_service_name`, which defaults to "vertica", and `kerberos_host_name`, which defaults to the value of argument `host`. For example, From b124af19bd8de1626ca6314dccae31a95147e14f Mon Sep 17 00:00:00 2001 From: Siting Ren Date: Wed, 26 Jun 2024 11:49:06 +0800 Subject: [PATCH 20/22] update test --- README.md | 2 +- vertica_python/tests/unit_tests/test_parsedsn.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c46bc65..085d4840 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ There are two options to control client-server TLS: `tlsmode` and `ssl`. If both | 'prefer' | (not set) | (Default) first try a TLS connection; if TLS is disabled on the server, then fallback to a non-TLS connection.
Note: If TLS is enabled on the server and TLS connection fails, the client rejects the connection. | | 'require' | True | connects using TLS without verifying certificates. If the TLS connection attempt fails, the client rejects the connection. | | 'verify-ca' || connects using TLS and confirms that the server certificate has been signed by the certificate authority. | -| 'verify-full' ||connects using TLS, confirms that the server certificate has been signed by the certificate authority, and verifies that the host name matches the name provided in the server certificate. | +| 'verify-full' || connects using TLS, confirms that the server certificate has been signed by the certificate authority, and verifies that the host name matches the name provided in the server certificate. | When `tlsmode` is 'verify-ca' or 'verify-full', these options take certificate/key files: `tls_cafile`, `tls_certfile` and `tls_keyfile`. Otherwise, these options are ignored. diff --git a/vertica_python/tests/unit_tests/test_parsedsn.py b/vertica_python/tests/unit_tests/test_parsedsn.py index b87aab4f..6a13256d 100644 --- a/vertica_python/tests/unit_tests/test_parsedsn.py +++ b/vertica_python/tests/unit_tests/test_parsedsn.py @@ -43,7 +43,8 @@ def test_str_arguments(self): 'log_path=/home/admin/vClient.log&log_level=DEBUG&' 'oauth_access_token=GciOiJSUzI1NiI&' 'workload=python_test_workload&tlsmode=verify-ca&' - 'tls_cafile=tls/ca_cert.pem&' + 'tls_cafile=tls/ca_cert.pem&tls_certfile=tls/cert.pem&' + 'tls_keyfile=tls/key.pem&' 'kerberos_service_name=krb_service&kerberos_host_name=krb_host') expected = {'database': 'db1', 'host': 'localhost', 'user': 'john', 'password': 'pwd', 'port': 5433, 'log_level': 'DEBUG', @@ -53,6 +54,8 @@ def test_str_arguments(self): 'workload': 'python_test_workload', 'tlsmode': 'verify-ca', 'tls_cafile': 'tls/ca_cert.pem', + 'tls_certfile': 'tls/cert.pem', + 'tls_keyfile': 'tls/key.pem', 'kerberos_service_name': 'krb_service', 'kerberos_host_name': 'krb_host'} parsed = parse_dsn(dsn) From 82da3a8e54611ca792df81f75a517f473fc30fd9 Mon Sep 17 00:00:00 2001 From: sitingren Date: Thu, 4 Jul 2024 09:21:34 +0000 Subject: [PATCH 21/22] Update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 085d4840..c988cd68 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ with vertica_python.connect(**conn_info) as connection: | session_label | Sets a label for the connection on the server. This value appears in the client_label column of the _v_monitor.sessions_ system table.
**_Default_**: an auto-generated label with format of `vertica-python-{version}-{random_uuid}` | | ssl | See [TLS/SSL](#tlsssl).
**_Default_**: None (tlsmode="prefer") | | tlsmode | Controls whether the connection to the server uses TLS encryption.
See [TLS/SSL](#tlsssl).
**_Default_**: "prefer" | -| tls_cafile | The name of a file containing SSL certificate authority (CA) certificate(s).
See [TLS/SSL](#tlsssl). | +| tls_cafile | The name of a file containing trusted SSL certificate authority (CA) certificate(s).
See [TLS/SSL](#tlsssl). | | tls_certfile | The name of a file containing client's certificate(s).
See [TLS/SSL](#tlsssl). | | tls_keyfile | The name of a file containing client's private key.
See [TLS/SSL](#tlsssl). | | unicode_error | See [UTF-8 encoding issues](#utf-8-encoding-issues).
**_Default_**: 'strict' (throw error on invalid UTF-8 results) | @@ -155,8 +155,8 @@ There are two options to control client-server TLS: `tlsmode` and `ssl`. If both | 'disable' | False | only try a non-TLS connection. | | 'prefer' | (not set) | (Default) first try a TLS connection; if TLS is disabled on the server, then fallback to a non-TLS connection.
Note: If TLS is enabled on the server and TLS connection fails, the client rejects the connection. | | 'require' | True | connects using TLS without verifying certificates. If the TLS connection attempt fails, the client rejects the connection. | -| 'verify-ca' || connects using TLS and confirms that the server certificate has been signed by the certificate authority. | -| 'verify-full' || connects using TLS, confirms that the server certificate has been signed by the certificate authority, and verifies that the host name matches the name provided in the server certificate. | +| 'verify-ca' || connects using TLS and confirms that the server certificate has been signed by a trusted certificate authority. | +| 'verify-full' || connects using TLS, confirms that the server certificate has been signed by a trusted certificate authority, and verifies that the host name matches the name provided in the server certificate. | When `tlsmode` is 'verify-ca' or 'verify-full', these options take certificate/key files: `tls_cafile`, `tls_certfile` and `tls_keyfile`. Otherwise, these options are ignored. From c1df49d7e689def3b2bebc02210a31c0774c93d2 Mon Sep 17 00:00:00 2001 From: sitingren Date: Thu, 4 Jul 2024 10:06:10 +0000 Subject: [PATCH 22/22] wrap function --- vertica_python/vertica/connection.py | 57 ++++++++++++++++------------ 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py index 15bc9446..92baf48c 100644 --- a/vertica_python/vertica/connection.py +++ b/vertica_python/vertica/connection.py @@ -496,6 +496,8 @@ def _socket(self): # modify the socket connection based on client connection options try: + ssl_context, force = self._generate_ssl_context() + # enable load balancing load_balance_options = self.options.get('connection_load_balance') self._logger.debug('Connection load balance option is {0}'.format( @@ -504,30 +506,8 @@ def _socket(self): raw_socket = self.balance_load(raw_socket) # enable TLS - tlsmode_options = self.options.get('tlsmode') - ssl_options = self.options.get('ssl') - # If TLSmode option and SSL option are set, TLSmode option takes precedence. - ssl_context = None - if tlsmode_options is not None: - tlsmode = TLSMode(tlsmode_options) - elif ssl_options is not None: - if isinstance(ssl_options, ssl.SSLContext): - ssl_context = ssl_options - tlsmode = TLSMode.REQUIRE # placeholder - elif isinstance(ssl_options, bool): - tlsmode = TLSMode.REQUIRE if ssl_options else TLSMode.DISABLE - else: - raise TypeError('The value of connection option "ssl" should be a bool or ssl.SSLContext object') - else: - tlsmode = TLSMode(DEFAULT_TLSMODE) - self._logger.debug(f'Connection TLS Mode is {tlsmode.name}') - if tlsmode.requests_encryption(): - if ssl_context is None: - cafile = self.options.get('tls_cafile') - certfile = self.options.get('tls_certfile') - keyfile = self.options.get('tls_keyfile') - ssl_context = tlsmode.get_sslcontext(cafile, certfile, keyfile) - raw_socket = self.enable_ssl(raw_socket, ssl_context, force=tlsmode.requires_encryption()) + if ssl_context is not None: + raw_socket = self.enable_ssl(raw_socket, ssl_context, force=force) except: self._logger.debug('Close the socket') raw_socket.close() @@ -536,6 +516,35 @@ def _socket(self): self.socket = raw_socket return self.socket + def _generate_ssl_context(self): + tlsmode_options = self.options.get('tlsmode') + ssl_options = self.options.get('ssl') + # If TLSmode option and SSL option are set, TLSmode option takes precedence. + ssl_context = None + if tlsmode_options is not None: + tlsmode = TLSMode(tlsmode_options) + elif ssl_options is not None: + if isinstance(ssl_options, ssl.SSLContext): + ssl_context = ssl_options + tlsmode = TLSMode.REQUIRE # placeholder + elif isinstance(ssl_options, bool): + tlsmode = TLSMode.REQUIRE if ssl_options else TLSMode.DISABLE + else: + raise TypeError('The value of connection option "ssl" should be a bool or ssl.SSLContext object') + else: + tlsmode = TLSMode(DEFAULT_TLSMODE) + self._logger.debug(f'Connection TLS Mode is {tlsmode.name}') + + if tlsmode.requests_encryption(): + if ssl_context is None: + cafile = self.options.get('tls_cafile') + certfile = self.options.get('tls_certfile') + keyfile = self.options.get('tls_keyfile') + ssl_context = tlsmode.get_sslcontext(cafile, certfile, keyfile) + return ssl_context, tlsmode.requires_encryption() + else: + return None, False + def _socket_as_file(self): if self.socket_as_file is None: self.socket_as_file = self._socket().makefile('rb')