Include Demo deployment tests

.github/workflows/dev-pytest.yml

@@ -3,28 +3,32 @@ name: dev-pytest
 on:
   push:
     branches:
-      - dev
+      - dev  
 
+  workflow_dispatch: 
+
   pull_request:
     types: [opened, reopened]
 
-  workflow_dispatch:
-
 jobs:
   build:
-
+    env:
+      SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }}
+      SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }}
+      SNOWFLAKE_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_CONNECTIONS_MYCONNECTION_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }}
+      SNOWFLAKE_CONNECTIONS_MYCONNECTION_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_CONNECTIONS_MYCONNECTION_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.11"
       - name: Update pip
         run: |
           python -m pip install --upgrade pip
@@ -34,3 +38,36 @@ jobs:
       - name: Test with pytest
         run: |
           pytest
+      - name: Setup SnowCLI connection file
+        run: |
+          echo "[connections]" > config.toml
+          echo "[connections.myconnection]" >> config.toml
+          chmod 0600 config.toml
+      - name: Install snowcli
+        run: |
+          pip install snowflake-cli-labs
+      - name: Test Default snowcli connection
+        run: |
+          snow --version
+          echo "::group::Config Contents"
+          more config.toml
+          echo "::endgroup::"
+          snow --config-file config.toml connection test -c myconnection
+          echo "Tested SnowCLI connection successfully!"
+          echo "Set default connection"
+          snow --config-file config.toml connection set-default "myconnection"
+      - name: Test Basic Schemachange
+        run: |
+          snow --config-file config.toml sql -f ./demo/setup/setup_basics_demo.sql
+          schemachange --config-folder ./demo/basics
+          snow --config-file config.toml sql -f ./demo/teardown/teardown_basics_demo.sql
+      - name: Test Citibike Demo
+        run: |
+          snow --config-file config.toml sql -f ./demo/setup/setup_citibike_demo.sql
+          schemachange --config-folder ./demo/citibike
+          snow --config-file config.toml sql -f ./demo/teardown/teardown_citibike_demo.sql
+      - name: Test Citibike Demo Jinja Template
+        run: |
+          snow --config-file config.toml sql -f ./demo/setup/setup_citibike_demo_jinja.sql
+          schemachange --config-folder ./demo/citibike_jinja
+          snow --config-file config.toml sql -f ./demo/teardown/teardown_citibike_demo_jinja.sql

.github/workflows/master-pytest.yml

@@ -10,7 +10,13 @@ on:
 
 jobs:
   build:
-
+    env:
+      SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }}
+      SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }}
+      SNOWFLAKE_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_CONNECTIONS_MYCONNECTION_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }}
+      SNOWFLAKE_CONNECTIONS_MYCONNECTION_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_CONNECTIONS_MYCONNECTION_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -33,3 +39,36 @@ jobs:
       - name: Test with pytest
         run: |
           pytest
+      - name: Setup SnowCLI connection file
+        run: |
+          echo "[connections]" > config.toml
+          echo "[connections.myconnection]" >> config.toml
+          chmod 0600 config.toml
+      - name: Install snowcli
+        run: |
+          pip install snowflake-cli-labs
+      - name: Test Default snowcli connection
+        run: |
+          snow --version
+          echo "::group::Config Contents"
+          more config.toml
+          echo "::endgroup::"
+          snow --config-file config.toml connection test -c myconnection
+          echo "Tested SnowCLI connection successfully!"
+          echo "Set default connection"
+          snow --config-file config.toml connection set-default "myconnection"
+      - name: Test Basic Schemachange
+        run: |
+          snow --config-file config.toml sql -f ./demo/setup/setup_basics_demo.sql
+          schemachange --config-folder ./demo/basics
+          snow --config-file config.toml sql -f ./demo/teardown/teardown_basics_demo.sql
+      - name: Test Citibike Demo
+        run: |
+          snow --config-file config.toml sql -f ./demo/setup/setup_citibike_demo.sql
+          schemachange --config-folder ./demo/citibike
+          snow --config-file config.toml sql -f ./demo/teardown/teardown_citibike_demo.sql
+      - name: Test Citibike Demo Jinja Template
+        run: |
+          snow --config-file config.toml sql -f ./demo/setup/setup_citibike_demo_jinja.sql
+          schemachange --config-folder ./demo/citibike_jinja
+          snow --config-file config.toml sql -f ./demo/teardown/teardown_citibike_demo_jinja.sql

demo/README.MD

@@ -0,0 +1,9 @@
+# PROVISION
+
+# Setup
+
+# Teardown
+
+# citibike
+
+# citibike jinja

demo/basics/A__basic001.sql

@@ -0,0 +1 @@
+SELECT 1;

demo/basics/R__basic001.sql

@@ -0,0 +1 @@
+SELECT 1;

demo/basics/schemachange-config.yml

@@ -0,0 +1,16 @@
+config-version: 1
+
+root-folder: ./demo/basics
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: SCHEMACHANGE_DEMO_DEPLOY
+snowflake-warehouse: SCHEMACHANGE_DEMO_WH
+snowflake-database: SCHEMACHANGE_DEMO
+snowflake-schema: BASICS_DEMO
+change-history-table: "SCHEMACHANGE_DEMO.SCHEMACHANGE.BASICS_DEMO_CHANGE_HISTORY"
+create-change-history-table: true
+
+vars:
+  database_name: {{env_var('SNOWFLAKE_DATABASE', 'SCHEMACHANGE_DEMO')}}
+  schema_name: {{env_var('SNOWFLAKE_SCHEMA', 'BASICS_DEMO')}}

demo/citibike/A__checks.sql

@@ -0,0 +1 @@
+SELECT 1;

demo/citibike/R__checks.sql

@@ -0,0 +1,2 @@
+-- Using Default context
+SELECT 1;

demo/citibike/V1.1__initial_database_objects.sql

@@ -1,8 +1,7 @@
--- Create the database if it doesn't exist
-CREATE DATABASE IF NOT EXISTS SCHEMACHANGE_DEMO;
-
+-- Use Demo Database if it doesn't exist
+USE DATABASE SCHEMACHANGE_DEMO;
 -- Set the database and schema context
-USE SCHEMA SCHEMACHANGE_DEMO.PUBLIC;
+USE SCHEMA SCHEMACHANGE_DEMO.CITIBIKE_DEMO;
 
 -- Create the file formats
 CREATE OR REPLACE FILE FORMAT CSV_NO_HEADER

demo/citibike/V1.2__load_tables_from_s3.sql

@@ -1,9 +1,12 @@
 -- Set the database and schema context
-USE SCHEMA SCHEMACHANGE_DEMO.PUBLIC;
+USE SCHEMA SCHEMACHANGE_DEMO.CITIBIKE_DEMO;
 
 -- Load the trips data
+-- Trips data bucket content has been updated. 
+-- https://stackoverflow.com/questions/72235656/snowflake-trial-data-in-wrong-format
 COPY INTO TRIPS FROM @TRIPS
-    FILE_FORMAT = (FORMAT_NAME = 'CSV_NO_HEADER');
+    FILE_FORMAT = (FORMAT_NAME = 'CSV_NO_HEADER')
+    PATTERN = '.*trips_.*csv.gz';
 
 -- Load the weather data
 COPY INTO WEATHER FROM

demo/citibike/schemachange-config.yml

@@ -0,0 +1,13 @@
+config-version: 1
+
+root-folder: ./demo/citibike
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: SCHEMACHANGE_DEMO_DEPLOY
+snowflake-warehouse: SCHEMACHANGE_DEMO_WH
+snowflake-database: SCHEMACHANGE_DEMO
+snowflake-schema: CITIBIKE_DEMO
+change-history-table: "SCHEMACHANGE_DEMO.SCHEMACHANGE.CITIBIKE_DEMO_CHANGE_HISTORY"
+create-change-history-table: true
+

demo/citibike_jinja/schemachange-config.yml

@@ -1,9 +1,20 @@
 config-version: 1
 
-root-folder: scripts
-modules-folder: modules
+root-folder: ./demo/citibike_jinja/scripts
+modules-folder: ./demo/citibike_jinja/modules
+
+snowflake-user: {{ env_var('SNOWFLAKE_USER')}}
+snowflake-account: {{ env_var('SNOWFLAKE_ACCOUNT')}}
+snowflake-role: SCHEMACHANGE_DEMO_DEPLOY
+snowflake-warehouse: SCHEMACHANGE_DEMO_WH
+snowflake-database: SCHEMACHANGE_DEMO
+snowflake-schema: CITIBIKE_DEMO_JINJA
+change-history-table: "SCHEMACHANGE_DEMO.SCHEMACHANGE.CITIBIKE_DEMO_JNJA_CHANGE_HISTORY"
+create-change-history-table: true
+
 vars:
-  database_name: {{env_var('SF_DATABASE', 'SCHEMACHANGE_DEMO_JINJA')}}
+  database_name: {{env_var('SNOWFLAKE_DATABASE', 'SCHEMACHANGE_DEMO')}}
+  schema_name: {{env_var('SNOWFLAKE_SCHEMA', 'CITIBIKE_DEMO_JINJA')}}
   secrets:
     # not a good example of secrets, just here to demo the secret filtering
     trips_s3_bucket: s3://snowflake-workshop-lab/citibike-trips

demo/citibike_jinja/scripts/V1.1__initial_database_objects.sql

@@ -1,9 +1,9 @@
 {% from 'modules/create_stage.j2' import create_stage-%}
 -- Create the database if it doesn't exist
-CREATE DATABASE IF NOT EXISTS {{database_name}};
+USE DATABASE {{database_name}};
 
 -- Set the database and schema context
-USE SCHEMA {{database_name}}.PUBLIC;
+USE SCHEMA {{database_name}}.{{schema_name}};
 
 -- Create the file formats
 CREATE OR REPLACE FILE FORMAT CSV_NO_HEADER

demo/citibike_jinja/scripts/V1.2__load_tables_from_s3.sql

@@ -1,9 +1,10 @@
 -- Set the database and schema context
-USE SCHEMA {{database_name}}.PUBLIC;
+USE SCHEMA {{database_name}}.{{schema_name}};
 
 -- Load the trips data
 COPY INTO TRIPS FROM @TRIPS
-    FILE_FORMAT = (FORMAT_NAME = 'CSV_NO_HEADER');
+    FILE_FORMAT = (FORMAT_NAME = 'CSV_NO_HEADER')
+    PATTERN = '.*trips_.*csv.gz';
 
 -- Load the weather data
 COPY INTO WEATHER FROM

demo/provision/initialize.sql

@@ -0,0 +1,47 @@
+-- This script is provided as a sample setup to use database roles, warehouse, admin role, deploy role as an example. 
+-- YOu may choose to have your own RBAC and SCHEMACHANGE database setup depending on your organization objectives.
+-- Set these to personalize your deployment
+SET SERVICE_USER_PASSWORD = 'CHANGEME';
+SET ADMIN_USER = 'CHANGEME';
+SET TARGET_DB_NAME = 'SCHEMACHANGE_DEMO'; -- Name of database that will have the SCHEMACHANGE Schema for change tracking.
+
+-- Dependent Variables; Change the naming pattern if you want but not necessary
+SET ADMIN_ROLE = $TARGET_DB_NAME || '_ADMIN'; -- This role will own the database and schemas.
+SET DEPLOY_ROLE = $TARGET_DB_NAME || '_DEPLOY'; -- This role will be granted privileges to create objects in any schema in the database
+SET SERVICE_USER = $TARGET_DB_NAME || '_SVC_USER'; -- This user will be granted the Deploy role.
+SET WAREHOUSE_NAME = $TARGET_DB_NAME || '_WH';
+SET AC_U = '_AC_U_' || $WAREHOUSE_NAME;
+SET AC_O = '_AC_O_' || $WAREHOUSE_NAME;
+
+USE ROLE USERADMIN;
+-- Service user used to run SCHEMACHANGE deployments
+CREATE USER IF NOT EXISTS IDENTIFIER($SERVICE_USER) WITH PASSWORD=$SERVICE_USER_PASSWORD MUST_CHANGE_PASSWORD=FALSE;
+-- Role granted to a human user to manage the database permissions and database roles.
+CREATE ROLE IF NOT EXISTS IDENTIFIER($ADMIN_ROLE);
+CREATE ROLE IF NOT EXISTS IDENTIFIER($DEPLOY_ROLE);
+CREATE ROLE IF NOT EXISTS IDENTIFIER($AC_U);
+CREATE ROLE IF NOT EXISTS IDENTIFIER($AC_O);
+GRANT ROLE IDENTIFIER($AC_U) TO ROLE IDENTIFIER($AC_O);
+
+
+-- Role hierarchy tied to SYSADMIN;
+USE ROLE SECURITYADMIN;
+GRANT ROLE IDENTIFIER($DEPLOY_ROLE) TO ROLE IDENTIFIER($ADMIN_ROLE);
+GRANT ROLE IDENTIFIER($ADMIN_ROLE) TO ROLE SYSADMIN;
+
+GRANT ROLE IDENTIFIER($ADMIN_ROLE) TO USER IDENTIFIER($SERVICE_USER);
+GRANT ROLE IDENTIFIER($ADMIN_ROLE) TO USER IDENTIFIER($ADMIN_USER);
+
+USE ROLE SYSADMIN;
+CREATE DATABASE IF NOT EXISTS IDENTIFIER($TARGET_DB_NAME);
+
+USE ROLE SECURITYADMIN;
+GRANT OWNERSHIP ON DATABASE IDENTIFIER($TARGET_DB_NAME) TO ROLE IDENTIFIER($ADMIN_ROLE) WITH GRANT OPTION;
+
+USE ROLE SYSADMIN;
+CREATE WAREHOUSE IF NOT EXISTS IDENTIFIER($WAREHOUSE_NAME);
+USE ROLE SECURITYADMIN;
+GRANT OWNERSHIP ON WAREHOUSE IDENTIFIER($WAREHOUSE_NAME) TO ROLE IDENTIFIER($ADMIN_ROLE) WITH GRANT OPTION;
+GRANT USAGE ON WAREHOUSE IDENTIFIER($WAREHOUSE_NAME) TO ROLE IDENTIFIER($AC_U);
+GRANT OPERATE ON WAREHOUSE IDENTIFIER($WAREHOUSE_NAME) TO ROLE IDENTIFIER($AC_O);
+GRANT ROLE IDENTIFIER($AC_U) TO ROLE IDENTIFIER($DEPLOY_ROLE);

demo/provision/setup_schemachange_schema.sql

@@ -0,0 +1,65 @@
+SET TARGET_SCHEMA_NAME = 'SCHEMACHANGE';
+SET TARGET_DB_NAME = 'SCHEMACHANGE_DEMO'; -- Name of database that will have the SCHEMACHANGE Schema for change tracking.
+-- Dependent Variables; Change the naming pattern if you want but not necessary
+SET ADMIN_ROLE = $TARGET_DB_NAME || '_ADMIN'; -- This role will own the database and schemas.
+SET DEPLOY_ROLE = $TARGET_DB_NAME || '_DEPLOY'; -- This role will be granted privileges to create objects in any schema in the database
+SET WAREHOUSE_NAME = $TARGET_DB_NAME || '_WH';
+SET SCHEMACHANGE_NAMESPACE = $TARGET_DB_NAME || '.' || $TARGET_SCHEMA_NAME;
+SET SC_M = 'SC_M_' || $TARGET_SCHEMA_NAME;
+SET SC_R = 'SC_R_' || $TARGET_SCHEMA_NAME;
+SET SC_W = 'SC_W_' || $TARGET_SCHEMA_NAME;
+SET SC_C = 'SC_C_' || $TARGET_SCHEMA_NAME;
+
+USE ROLE IDENTIFIER($ADMIN_ROLE);
+USE DATABASE IDENTIFIER($TARGET_DB_NAME);
+USE WAREHOUSE IDENTIFIER($WAREHOUSE_NAME);
+
+CREATE DATABASE ROLE IF NOT EXISTS DB_M;
+CREATE DATABASE ROLE IF NOT EXISTS DB_R;
+CREATE DATABASE ROLE IF NOT EXISTS DB_W;
+CREATE DATABASE ROLE IF NOT EXISTS DB_C;
+
+GRANT DATABASE ROLE DB_C TO ROLE IDENTIFIER($DEPLOY_ROLE);
+
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_M);
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_R);
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_W);
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_C);
+
+GRANT DATABASE ROLE IDENTIFIER($SC_M) TO DATABASE ROLE DB_M;
+GRANT DATABASE ROLE IDENTIFIER($SC_R) TO DATABASE ROLE DB_R;
+GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE DB_W;
+GRANT DATABASE ROLE IDENTIFIER($SC_C) TO DATABASE ROLE DB_C;
+GRANT DATABASE ROLE IDENTIFIER($SC_M) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT DATABASE ROLE IDENTIFIER($SC_R) TO DATABASE ROLE IDENTIFIER($SC_W);
+GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C);
+
+CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS;
+-- USE SCHEMA INFORMATION_SCHEMA;
+-- DROP SCHEMA IF EXISTS PUBLIC;
+
+USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE);
+-- SCHEMA
+-- SC_M
+GRANT USAGE ON DATABASE IDENTIFIER($TARGET_DB_NAME) TO DATABASE ROLE IDENTIFIER($SC_M);
+GRANT USAGE ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+-- SC_R
+GRANT MONITOR ON DATABASE IDENTIFIER($TARGET_DB_NAME) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT MONITOR ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+-- SC_W 
+-- None 
+-- SC_C
+GRANT MODIFY, APPLYBUDGET, ADD SEARCH OPTIMIZATION ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C);
+
+-- TABLES
+-- SC_M
+GRANT REFERENCES ON ALL TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+GRANT REFERENCES ON FUTURE TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+-- SC_R
+GRANT SELECT ON ALL TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT SELECT ON FUTURE TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+-- SC_W
+GRANT INSERT, UPDATE, DELETE, TRUNCATE, EVOLVE SCHEMA ON ALL TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_W);
+GRANT INSERT, UPDATE, DELETE, TRUNCATE, EVOLVE SCHEMA ON FUTURE TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_W);
+-- SC_C
+GRANT CREATE TABLE ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C);