feat: tests, docs, linting, remove pandas dep

.github/workflows/dev-pytest.yml

@@ -1,32 +1,88 @@
 name: dev-pytest
 
-on:
-  push:
-    branches:
-      - dev
+defaults:
+  run:
+    shell: bash
 
+on:
   workflow_dispatch:
 
+  pull_request:
+    types:
+      - labeled
+    branches:
+      - dev
 
 jobs:
   build:
-
-    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja']
+        os: ["ubuntu-latest"]
+        python-version: ["3.11"]
+    runs-on: ${{ matrix.os }}
+    if: ${{ github.event.label.name == 'ci-run-tests' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
 
+    env:
+      SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }}
+      SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }}
+      SNOWFLAKE_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_DATABASE: SCHEMACHANGE_DEMO
+      SNOWFLAKE_WAREHOUSE: SCHEMACHANGE_DEMO_WH
+      SNOWFLAKE_ROLE: '"SCHEMACHANGE_DEMO-DEPLOY"'
+      MY_TARGET_SCHEMA: ${{ matrix.scenario-name }}_${{ github.run_number }}_${{ strategy.job-index }}
+      SCENARIO_NAME: ${{ matrix.scenario-name }}
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-      - name: Update pip
-        run: |
-          python -m pip install --upgrade pip
+          python-version: ${{ matrix.python-version }}
+      # The next few steps (Install dependencies, Lint with Flake8, Test with Pytest) will install Python dependencies,
+      # run tests and lint with a variety of Python versions
+      # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
       - name: Install dependencies
         run: |
+          python -m pip install --upgrade pip
+          python -m pip install flake8 pytest
           python -m pip install -e .[dev]
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Test with pytest
+        id: pytest
         run: |
           pytest
+      # Testing Schemachange demo projects
+      - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema
+        run: |
+          echo "::group::Setting up ${MY_TARGET_SCHEMA}"
+          schemachange deploy --config-folder ./demo/setup/${SCENARIO_NAME}
+          echo "::endgroup::"
+          echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}"
+          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/A__render.sql
+          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/R__render.sql
+          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/V1.0.0__render.sql
+          echo "::endgroup::"
+          echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}"
+          set +e
+          schemachange deploy --config-folder ./demo/${SCENARIO_NAME}
+          RESULT=$?
+          if [ $RESULT -eq 0 ]; then
+            echo "Deployment Completed!"
+          else
+            echo "Deployment Failed. Proceeding to Teardown."
+          fi
+          echo "::endgroup::"
+          set -e
+          echo "::group::Tearing down up ${MY_TARGET_SCHEMA}"
+          schemachange deploy --config-folder ./demo/teardown/${SCENARIO_NAME}
+          echo "::endgroup::"
+          if [ $RESULT -ne 0 ]; then
+            exit 1
+          fi

.github/workflows/master-pytest.yml

@@ -0,0 +1,92 @@
+name: master-pytest
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    types:
+      - labeled
+    branches:
+      - main
+
+  push:
+    tags:
+      - "v*" # Push events to matching v*, i.e. v3.6.2, v4.0.0
+
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja']
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    runs-on: ${{ matrix.os }}
+    if: ${{ github.event.label.name == 'ci-run-tests' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+    env:
+      SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }}
+      SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }}
+      SNOWFLAKE_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_DATABASE: SCHEMACHANGE_DEMO
+      SNOWFLAKE_WAREHOUSE: SCHEMACHANGE_DEMO_WH
+      SNOWFLAKE_ROLE: '"SCHEMACHANGE_DEMO-DEPLOY"'
+      MY_TARGET_SCHEMA: ${{ matrix.scenario-name }}_${{ github.run_number }}_${{ strategy.job-index }}
+      SCENARIO_NAME: ${{ matrix.scenario-name }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      # The next few steps (Install dependencies, Lint with Flake8, Test with Pytest) will install Python dependencies,
+      # run tests and lint with a variety of Python versions
+      # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install flake8 pytest
+          python -m pip install -e .[dev]
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        id: pytest
+        run: |
+          pytest
+      # Testing Schemachange demo projects
+      - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema
+        run: |
+          echo "::group::Setting up ${MY_TARGET_SCHEMA}"
+          schemachange deploy --config-folder ./demo/setup/${SCENARIO_NAME}
+          echo "::endgroup::"
+          echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}"
+          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/A__render.sql
+          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/R__render.sql
+          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/V1.0.0__render.sql
+          echo "::endgroup::"
+          echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}"
+          set +e
+          schemachange deploy --config-folder ./demo/${SCENARIO_NAME}
+          RESULT=$?
+          if [ $RESULT -eq 0 ]; then
+            echo "Deployment Completed!"
+          else
+            echo "Deployment Failed. Proceeding to Teardown."
+          fi
+          echo "::endgroup::"
+          set -e
+          echo "::group::Tearing down up ${MY_TARGET_SCHEMA}"
+          schemachange deploy --config-folder ./demo/teardown/${SCENARIO_NAME}
+          echo "::endgroup::"
+          if [ $RESULT -ne 0 ]; then
+            exit 1
+          fi

.github/workflows/python-publish.yml

@@ -21,9 +21,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies

.pre-commit-config.yaml

@@ -1,12 +1,43 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
--   repo: https://github.com/asottile/setup-cfg-fmt
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-ast
+    - id: check-case-conflict
+    - id: check-docstring-first
+    - id: check-merge-conflict
+    - id: check-toml
+    - id: check-yaml
+    - id: name-tests-test
+  - repo: https://github.com/asottile/setup-cfg-fmt
     rev: v2.4.0
     hooks:
-    -   id: setup-cfg-fmt
+    - id: setup-cfg-fmt
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.2
+    hooks:
+    - id: pyupgrade
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.3.4
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+  - repo: git@github.com:GitGuardian/ggshield.git
+    rev: v1.28.0
+    hooks:
+      - id: ggshield
+        language_version: python3
+        stages: [commit]
+  - repo: https://github.com/PyCQA/flake8
+    rev: '6.0.0'
+    hooks:
+        - id: flake8
+          args: [ '--max-line-length=88', '--extend-ignore=E203,E501,W503,E266', '--exclude=.venv' ]

CHANGELOG.md

@@ -3,7 +3,19 @@ All notable changes to this project will be documented in this file.
 
 *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).*
 
-## [3.6.2] - 2024-06-10
+## [3.7.0] - 2024-07-31
+### Added
+- Improved unit test coverage
+- Added Session ID as part of the initial connection successful message to be visible in the logs
+### Changed
+- Aligning with snowflake [identifier requirements](https://docs.snowflake.com/en/sql-reference/identifiers-syntax) in the configuration settings
+- Fixed the bug with `Missing default warehouse`
+- Fixed Demo examples resulting from change in public data set location
+- Removed pandas library dependency to improve schemachange install footprint
+- Updated Github Actions Workflow to check PRs and Merges does not break the demo examples.
+- Updated Docs related to latest Demo content included in schemachange
+
+## [3.6.2] - 2024-07-10
 ### Changed
 - Updated pandas version dependency
 - Pinned NumPy version dependency
@@ -93,12 +105,12 @@ All notable changes to this project will be documented in this file.
 ## [3.2.2] - 2021-11-06
 ### Added
 - Restored CLI tests, hopefully less fragile now.
-- Added Github CI workflow to run unit tests and a basic execution test.
+- Added GitHub CI workflow to run unit tests and a basic execution test.
 - `schemachange.cli.main` is now defined as `def main(argv: List[str]=sys.argv)`, to allow consumers to pass a list of arguments easily.
 
 ## [3.2.1] - 2021-11-04
 ### Fixed
-- Jinja Template Engine was not recognising scripts in subfolders on windows machines. Jinja was expecting the paths to follow a unix style ie SQL/V2.0.0__ADHOC_SCRIPT.sql but on windows machines this was being passed through as SQL\V2.0.0__ADHOC_SCRIPT.sql.
+- Jinja Template Engine was not recognising scripts in subfolders on Windows machines. Jinja was expecting the paths to follow a unix style ie SQL/V2.0.0__ADHOC_SCRIPT.sql but on Windows machines this was being passed through as SQL\V2.0.0__ADHOC_SCRIPT.sql.
 
 ### Removed
 - Removed fragile unit tests in test_main.py.
@@ -232,7 +244,7 @@ All notable changes to this project will be documented in this file.
 - Add the Snowflake account name to the script output to provide more log context
 
 ### Removed
-- Removed the deprecated `--snowflake-region` parameter. Instead use the `-a` or `--snowflake-account` account parameter. See [Usage Notes for the account Parameter (for the connect Method)](https://docs.snowflake.com/en/user-guide/python-connector-api.html#label-account-format-info) for more details on how to structure the account name.
+- Removed the deprecated `--snowflake-region` parameter. Instead, use the `-a` or `--snowflake-account` account parameter. See [Usage Notes for the account Parameter (for the connect Method)](https://docs.snowflake.com/en/user-guide/python-connector-api.html#label-account-format-info) for more details on how to structure the account name.
 
 
 ## [2.1.0] - 2020-05-26