From a8f4850af97915eff9f3af86568d291f72987c3b Mon Sep 17 00:00:00 2001 From: rosspb3 Date: Tue, 20 Feb 2024 18:40:46 -0600 Subject: [PATCH 1/3] Removed the pandas library as a requirement, since its usage was minimal. --- requirements.txt | 1 - schemachange/cli.py | 21 ++++++++++----------- setup.cfg | 1 - 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5e8d0440..a151cfb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ Jinja2~=3.0 -pandas~=1.3 PyYAML~=6.0 snowflake-connector-python>=2.8,<4.0 diff --git a/schemachange/cli.py b/schemachange/cli.py index 31745f43..49643231 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -17,7 +17,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from jinja2.loaders import BaseLoader -from pandas import DataFrame #region Global Variables # metadata @@ -398,21 +397,21 @@ def create_change_history_table_if_missing(self, change_history_table): query = self._q_ch_ddl_table.format(**change_history_table) self.execute_snowflake_query(query) - def fetch_r_scripts_checksum(self,change_history_table): + def fetch_r_scripts_checksum(self,change_history_table) -> Dict[str, str]: + """ + Fetches the checksum of the last executed R script from the change history table. + return: a dictionary with the script name as key and the last successfully installed script checksum as value + """ + # Note: Query only fetches last successfully installed checksum for R scripts query = self._q_ch_r_checksum.format(**change_history_table) results = self.execute_snowflake_query(query) # Collect all the results into a dict - d_script_checksum = DataFrame(columns=['script_name', 'checksum']) - script_names = [] - checksums = [] + d_script_checksum = {} for cursor in results: for row in cursor: - script_names.append(row[0]) - checksums.append(row[1]) + d_script_checksum[row[0]] = row[1] - d_script_checksum['script_name'] = script_names - d_script_checksum['checksum'] = checksums return d_script_checksum def fetch_change_history(self, change_history_table): @@ -560,8 +559,8 @@ def deploy_command(config): checksum_current = hashlib.sha224(content.encode('utf-8')).hexdigest() # check if R file was already executed - if (r_scripts_checksum is not None) and script_name in list(r_scripts_checksum['script_name']): - checksum_last = list(r_scripts_checksum.loc[r_scripts_checksum['script_name'] == script_name, 'checksum'])[0] + if r_scripts_checksum and (script_name in r_scripts_checksum): + checksum_last = r_scripts_checksum[script_name] else: checksum_last = '' diff --git a/setup.cfg b/setup.cfg index b3f8ec01..33fda483 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,7 +18,6 @@ classifiers = packages = schemachange install_requires = jinja2~=3.0 - pandas~=1.3 pyyaml~=6.0 snowflake-connector-python>=2.8,<4.0 python_requires = >=3.8 From 2f8b738a7d8f03d2a9126906e39b07c19d94a4e3 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 28 Mar 2024 00:26:24 -0400 Subject: [PATCH 2/3] resolving merge conflicts --- schemachange/cli.py | 101 ++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 287f3093..06a6dc92 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -354,21 +354,22 @@ def set_connection_args(self): if snowflake_authenticator.lower() == "oauth": oauth_token = self.get_oauth_token() - if self.verbose: - print( _log_auth_type % 'Oauth Access Token') - self.conArgs['token'] = oauth_token - self.conArgs['authenticator'] = 'oauth' - # External Browswer based SSO - elif snowflake_authenticator.lower() == 'externalbrowser': - self.conArgs['authenticator'] = 'externalbrowser' - if self.verbose: - print(_log_auth_type % 'External Browser') - # IDP based Authentication, limited to Okta - elif snowflake_authenticator.lower()[:8]=='https://': - - if self.verbose: - print(_log_auth_type % 'Okta') - print(_log_okta_ep % snowflake_authenticator) + if self.verbose: + print(_log_auth_type % "Oauth Access Token") + + self.conArgs["token"] = oauth_token + self.conArgs["authenticator"] = "oauth" + # External Browswer based SSO + elif snowflake_authenticator.lower() == "externalbrowser": + self.conArgs["authenticator"] = "externalbrowser" + if self.verbose: + print(_log_auth_type % "External Browser") + # IDP based Authentication, limited to Okta + elif snowflake_authenticator.lower()[:8] == "https://": + + if self.verbose: + print(_log_auth_type % "Okta") + print(_log_okta_ep % snowflake_authenticator) self.conArgs["password"] = snowflake_password self.conArgs["authenticator"] = snowflake_authenticator.lower() @@ -388,16 +389,16 @@ def set_connection_args(self): # default authenticator to snowflake self.conArgs["authenticator"] = default_authenticator - if self.conArgs['authenticator'].lower() == default_authenticator: - # Giving preference to password based authentication when both private key and password are specified. - if snowflake_password: - if self.verbose: - print(_log_auth_type % 'password' ) - self.conArgs['password'] = snowflake_password + if self.conArgs["authenticator"].lower() == default_authenticator: + # Giving preference to password based authentication when both private key and password are specified. + if snowflake_password: + if self.verbose: + print(_log_auth_type % "password") + self.conArgs["password"] = snowflake_password - elif os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH", ''): - if self.verbose: - print( _log_auth_type % 'private key') + elif os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH", ""): + if self.verbose: + print(_log_auth_type % "private key") private_key_password = os.getenv("SNOWFLAKE_PRIVATE_KEY_PASSPHRASE", "") if private_key_password: @@ -470,22 +471,22 @@ def create_change_history_table_if_missing(self, change_history_table): query = self._q_ch_ddl_table.format(**change_history_table) self.execute_snowflake_query(query) - def fetch_r_scripts_checksum(self,change_history_table) -> Dict[str, str]: - """ - Fetches the checksum of the last executed R script from the change history table. - return: a dictionary with the script name as key and the last successfully installed script checksum as value - """ - # Note: Query only fetches last successfully installed checksum for R scripts - query = self._q_ch_r_checksum.format(**change_history_table) - results = self.execute_snowflake_query(query) + def fetch_r_scripts_checksum(self, change_history_table) -> Dict[str, str]: + """ + Fetches the checksum of the last executed R script from the change history table. + return: a dictionary with the script name as key and the last successfully installed script checksum as value + """ + # Note: Query only fetches last successfully installed checksum for R scripts + query = self._q_ch_r_checksum.format(**change_history_table) + results = self.execute_snowflake_query(query) - # Collect all the results into a dict - d_script_checksum = {} - for cursor in results: - for row in cursor: - d_script_checksum[row[0]] = row[1] + # Collect all the results into a dict + d_script_checksum = {} + for cursor in results: + for row in cursor: + d_script_checksum[row[0]] = row[1] - return d_script_checksum + return d_script_checksum def fetch_change_history(self, change_history_table): query = self._q_ch_fetch.format(**change_history_table) @@ -674,18 +675,18 @@ def deploy_command(config): # Compute the checksum for the script checksum_current = hashlib.sha224(content.encode("utf-8")).hexdigest() - # check if R file was already executed - if r_scripts_checksum and (script_name in r_scripts_checksum): - checksum_last = r_scripts_checksum[script_name] - else: - checksum_last = '' - - # check if there is a change of the checksum in the script - if checksum_current == checksum_last: - if config["verbose"]: - print(_log_skip_r.format(**script)) - scripts_skipped += 1 - continue + # check if R file was already executed + if r_scripts_checksum and (script_name in r_scripts_checksum): + checksum_last = r_scripts_checksum[script_name] + else: + checksum_last = "" + + # check if there is a change of the checksum in the script + if checksum_current == checksum_last: + if config["verbose"]: + print(_log_skip_r.format(**script)) + scripts_skipped += 1 + continue print(_log_apply.format(**script)) if not config["dry_run"]: From 11bb3b0358b3a6795638147a60d586238292a4f4 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Thu, 28 Mar 2024 00:36:07 -0400 Subject: [PATCH 3/3] Fixed indents from merge conflict. --- schemachange/cli.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 06a6dc92..fb5558ef 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -354,25 +354,27 @@ def set_connection_args(self): if snowflake_authenticator.lower() == "oauth": oauth_token = self.get_oauth_token() - if self.verbose: - print(_log_auth_type % "Oauth Access Token") + if self.verbose: + print(_log_auth_type % "Oauth Access Token") - self.conArgs["token"] = oauth_token - self.conArgs["authenticator"] = "oauth" + self.conArgs["token"] = oauth_token + self.conArgs["authenticator"] = "oauth" # External Browswer based SSO - elif snowflake_authenticator.lower() == "externalbrowser": - self.conArgs["authenticator"] = "externalbrowser" - if self.verbose: - print(_log_auth_type % "External Browser") - # IDP based Authentication, limited to Okta - elif snowflake_authenticator.lower()[:8] == "https://": + elif snowflake_authenticator.lower() == "externalbrowser": + self.conArgs["authenticator"] = "externalbrowser" + if self.verbose: + print(_log_auth_type % "External Browser") + + # IDP based Authentication, limited to Okta + elif snowflake_authenticator.lower()[:8] == "https://": - if self.verbose: - print(_log_auth_type % "Okta") - print(_log_okta_ep % snowflake_authenticator) + if self.verbose: + print(_log_auth_type % "Okta") + print(_log_okta_ep % snowflake_authenticator) self.conArgs["password"] = snowflake_password self.conArgs["authenticator"] = snowflake_authenticator.lower() + elif snowflake_authenticator.lower() == "snowflake": self.conArgs["authenticator"] = default_authenticator # if authenticator is not a supported method or the authenticator variable is defined but not specified