Merge pull request #195 from Snowflake-Labs/dev

Code review for 3.6.0

.github/workflows/dependency-review.yml

@@ -0,0 +1,14 @@
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v3

.github/workflows/test.yml → .github/workflows/pytest.yml

@@ -1,29 +1,28 @@
-name: Python tests
+name: pytest
 
 on: [push]
 
 jobs:
   build:
 
+    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        github-runner: ['ubuntu-latest', 'windows-latest']
-
-    runs-on: ${{ matrix.github-runner }}
+        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Update pip
+        run: |
+          python -m pip install --upgrade pip
       - name: Install dependencies
         run: |
           python -m pip install -e .[dev]
       - name: Test with pytest
         run: |
           pytest
-      - name: Test run
-        run: |
-          schemachange --help

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+-   repo: https://github.com/asottile/setup-cfg-fmt
+    rev: v2.4.0
+    hooks:
+    -   id: setup-cfg-fmt

CHANGELOG.md

@@ -2,6 +2,19 @@
 All notable changes to this project will be documented in this file.
 
 *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).*
+
+## [3.6.0] - 2023-09-06
+### Changed
+- Fixed bug introduced in version 3.5.0 where the session state was not reset after a user script was run. This resulted in schemachange updates to the metadata table failing in some cases. schemachange will now reset the session back to the default settings after each user script is run
+- Updated the pytest GitHub Actions workflow
+- Cleaned up whitespace and formatting in files for consistency
+- Updated README file
+
+### Added
+- Added new dependency review GitHub Actions workflow
+- Added badges for pytest and PyPI
+
+
 ## [3.5.4] - 2023-09-01
 ### Changed
 - Fixed authentication workflow to check for authenticator type first, then for Key pair and finally default to password authentication.
@@ -29,7 +42,7 @@ All notable changes to this project will be documented in this file.
 - Cleaned up argument passing and other repetitive code using dictionary and set comparisons for easy maintenance. (Converted variable names to a consistent snake_case from a mix of kebab-case and snake_case)
 - Fixed change history table processing to allow mixed case names when '"' are used in the name.
 - Moved most error, log and warning messages and query strings to global or class variables.
-- Updated readme to cover new authentication methods 
+- Updated readme to cover new authentication methods
 
 ## [3.4.2] - 2022-10-24
 ### Changed

NOTICE

@@ -1,10 +1,9 @@
-This software includes the following python packages: 
- 
- Name                        License                  Author                                                              URL                                              
- Jinja2                      BSD License              Armin Ronacher                                                      https://palletsprojects.com/p/jinja/             
- PyYAML                      MIT License              Kirill Simonov                                                      https://pyyaml.org/                              
- pandas                      BSD License              The Pandas Development Team                                         https://pandas.pydata.org                        
+This software includes the following python packages:
+
+ Name                        License                  Author                                                              URL
+ Jinja2                      BSD License              Armin Ronacher                                                      https://palletsprojects.com/p/jinja/
+ PyYAML                      MIT License              Kirill Simonov                                                      https://pyyaml.org/
+ pandas                      BSD License              The Pandas Development Team                                         https://pandas.pydata.org
  pytest                      MIT License              Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt,                  https://docs.pytest.org/en/latest/
-                                                      Floris Bruynooghe, Brianna Laugher, Florian Bruhin and others  
- snowflake-connector-python  Apache Software License  Snowflake, Inc                                                      https://www.snowflake.com/                       
-
+                                                      Floris Bruynooghe, Brianna Laugher, Florian Bruhin and others
+ snowflake-connector-python  Apache Software License  Snowflake, Inc                                                      https://www.snowflake.com/

README.md

@@ -3,6 +3,8 @@
 
 *Looking for snowchange? You've found the right spot. snowchange has been renamed to schemachange.*
 
+[![pytest](https://github.com/Snowflake-Labs/schemachange/actions/workflows/pytest.yml/badge.svg)](https://github.com/Snowflake-Labs/schemachange/actions/workflows/pytest.yml)
+[![PyPI](https://img.shields.io/pypi/v/schemachange.svg)](https://pypi.org/project/schemachange)
 ## Overview
 
 schemachange is a simple python based tool to manage all of your [Snowflake](https://www.snowflake.com/) objects. It follows an Imperative-style approach to Database Change Management (DCM) and was inspired by the [Flyway database migration tool](https://flywaydb.org). When combined with a version control system and a CI/CD tool, database changes can be approved and deployed through a pipeline using modern software delivery practices. As such schemachange plays a critical role in enabling Database (or Data) DevOps.
@@ -69,7 +71,7 @@ schemachange expects a directory structure like the following to exist:
         |-- R__fn_sort_ascii.sql
 ```
 
-The schemachange folder structure is very flexible. The `project_root` folder is specified with the `-f` or `--root-folder` argument. Under the `project_root` folder you are free to arrange the change scripts any way you see fit. You can have as many subfolders (and nested subfolders) as you would like.
+The schemachange folder structure is very flexible. The `project_root` folder is specified with the `-f` or `--root-folder` argument. schemachange only pays attention to the filenames, not the paths. Therefore, under the `project_root` folder you are free to arrange the change scripts any way you see fit. You can have as many subfolders (and nested subfolders) as you would like.
 
 ## Change Scripts
 
@@ -111,7 +113,7 @@ e.g:
 All repeatable change scripts are applied each time the utility is run, if there is a change in the file.
 Repeatable scripts could be used for maintaining code that always needs to be applied in its entirety. e.g. stores procedures, functions and view definitions etc.
 
-Just like Flyway, within a single migration run, repeatable scripts are always applied after all pending versioned scripts have been executed. Repeatable scripts are applied in the order of their description.
+Just like Flyway, within a single migration run, repeatable scripts are always applied after all pending versioned scripts have been executed. Repeatable scripts are applied in alphabetical order of their description.
 
 ### Always Script Naming
 
@@ -228,7 +230,7 @@ Default [Password](https://docs.snowflake.com/en/user-guide/python-connector-exa
 If an authenticator is unsupported, then schemachange will default to `snowflake`. If the authenticator is `snowflake`, and both password and key pair values are provided then schemachange will use the password over the key pair values.
 
 ### Password Authentication
-The Snowflake user password for `SNOWFLAKE_USER` is required to be set in the environment variable `SNOWFLAKE_PASSWORD` prior to calling the script. schemachange will fail if the `SNOWFLAKE_PASSWORD` environment variable is not set. The environment variable `SNOWFLAKE_AUTHENTICATOR` will be set to `snowflake` if it not explicitly set. 
+The Snowflake user password for `SNOWFLAKE_USER` is required to be set in the environment variable `SNOWFLAKE_PASSWORD` prior to calling the script. schemachange will fail if the `SNOWFLAKE_PASSWORD` environment variable is not set. The environment variable `SNOWFLAKE_AUTHENTICATOR` will be set to `snowflake` if it not explicitly set.
 
 _**DEPRECATION NOTICE**: The `SNOWSQL_PWD` environment variable is deprecated but currently still supported. Support for it will be removed in a later version of schemachange. Please use `SNOWFLAKE_PASSWORD` instead._
 
@@ -242,20 +244,20 @@ The URL of the authenticator resource that will be receive the POST request.
 * token-response-name
 The Expected name of the JSON element containing the Token in the return response from the authenticator resource.
 * token-request-payload
-The Set of variables passed as a dictionary to the `data` element of the request. 
+The Set of variables passed as a dictionary to the `data` element of the request.
 * token-request-headers
-The Set of variables passed as a dictionary to the `headers` element of the request. 
+The Set of variables passed as a dictionary to the `headers` element of the request.
 
-It is recomended to use the YAML file and pass oauth secrets into the configuration using the templating engine instead of the command line option.  
+It is recomended to use the YAML file and pass oauth secrets into the configuration using the templating engine instead of the command line option.
 
 
 ### External Browser Authentication
-External browser authentication can be used for local development by setting the environment variable `SNOWFLAKE_AUTHENTICATOR` to the value `externalbrowser` prior to calling schemachange. 
+External browser authentication can be used for local development by setting the environment variable `SNOWFLAKE_AUTHENTICATOR` to the value `externalbrowser` prior to calling schemachange.
 The client will be prompted to authenticate in a browser that pops up. Refer to the [documentation](https://docs.snowflake.com/en/user-guide/admin-security-fed-auth-use.html#setting-up-browser-based-sso) to cache the token to minimize the number of times the browser pops up to authenticate the user.
 
 ### Okta Authentication
-For clients that do not have a browser, can use the popular SaaS Idp option to connect via Okta. This will require the Okta URL that you utilize for SSO. 
-Okta authentication can be used setting the environment variable `SNOWFLAKE_AUTHENTICATOR` to the value of your okta endpoint as a fully formed URL ( E.g. `https://<org_name>.okta.com`) prior to calling schemachange. 
+For clients that do not have a browser, can use the popular SaaS Idp option to connect via Okta. This will require the Okta URL that you utilize for SSO.
+Okta authentication can be used setting the environment variable `SNOWFLAKE_AUTHENTICATOR` to the value of your okta endpoint as a fully formed URL ( E.g. `https://<org_name>.okta.com`) prior to calling schemachange.
 
 _** NOTE**: Please disable Okta MFA for the user who uses Native SSO authentication with client drivers. Please consult your Okta administrator for more information._
 
@@ -330,14 +332,14 @@ dry-run: false
 # A string to include in the QUERY_TAG that is attached to every SQL statement executed
 query-tag: 'QUERY_TAG'
 
-# Information for Oauth token requests 
+# Information for Oauth token requests
 oauthconfig:
   # url Where token request are posted to
   token-provider-url: 'https://login.microsoftonline.com/{{ env_var('AZURE_ORG_GUID', 'default') }}/oauth2/v2.0/token'
   # name of Json entity returned by request
   token-response-name: 'access_token'
   # Headers needed for successful post or other security markings ( multiple labeled items permitted
-  token-request-headers: 
+  token-request-headers:
     Content-Type: "application/x-www-form-urlencoded"
     User-Agent: "python/schemachange"
   # Request Payload for Token (it is recommended pass

demo/citibike_jinja/modules/create_stage.j2

@@ -1,4 +1,4 @@
 {% macro create_stage(stage_name, URL) -%}
 CREATE OR REPLACE STAGE {{stage_name}}
     URL = '{{URL}}';
-{%- endmacro %}
+{%- endmacro %}

demo/citibike_jinja/schemachange-config.yml

@@ -8,4 +8,3 @@ vars:
     # not a good example of secrets, just here to demo the secret filtering
     trips_s3_bucket: s3://snowflake-workshop-lab/citibike-trips
     weather_s3_bucket: s3://snowflake-workshop-lab/weather-nyc
-

pyproject.toml

@@ -3,4 +3,4 @@ requires = [
   "setuptools >= 40.9.0",
   "wheel",
 ]
-build-backend = "setuptools.build_meta"
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -1,4 +1,4 @@
-snowflake-connector-python>=2.8,<4.0
+Jinja2~=3.0
 pandas~=1.3
 PyYAML~=6.0
-Jinja2~=3.0
+snowflake-connector-python>=2.8,<4.0