From edef65f3330f7ed099fd5e66b95e433141c84779 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 10 Jun 2020 09:53:42 -0600 Subject: [PATCH 1/5] do not create schemas --- core/dbt/task/run.py | 3 ++- core/dbt/task/runnable.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 40ad51585a3..61b9196cd6c 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -199,8 +199,9 @@ def print_results_line(self, results, execution_time): .format(stat_line=stat_line, execution=execution)) def before_run(self, adapter, selected_uids): - super().before_run(adapter, selected_uids) with adapter.connection_named('master'): + self.create_schemas(adapter, selected_uids) + self.populate_adapter_cache(adapter) self.safe_run_hooks(adapter, RunHookType.Start, {}) def after_run(self, adapter, results): diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 713d0d55cd9..d7c4fe7980d 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -352,7 +352,6 @@ def before_hooks(self, adapter): def before_run(self, adapter, selected_uids): with adapter.connection_named('master'): - self.create_schemas(adapter, selected_uids) self.populate_adapter_cache(adapter) def after_run(self, adapter, results): From a2de92d2a3cb5b1b5b4fe9842e979d4d5e50849a Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 10 Jun 2020 11:02:30 -0600 Subject: [PATCH 2/5] Fix azure pipelines builds by using the existing postgres setup --- azure-pipelines.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 88a2a174771..e4098a22b48 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,22 +32,18 @@ jobs: dependsOn: UnitTest steps: - - pwsh: echo "True" - displayName: Skip everything - pwsh: | - choco install postgresql --params '/Password:password' --params-global --version 10.6 - Set-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host all all ::1/128 trust" - Add-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host all all 127.0.0.1/32 trust" - # the service name is "postgresql-x64-10", conveniently it's both the display name and the actual name - Restart-Service postgresql-x64-10 - - & "C:\program files\postgresql\10\bin\createdb.exe" -U postgres dbt - & "C:\program files\postgresql\10\bin\psql.exe" -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';" - & "C:\program files\postgresql\10\bin\psql.exe" -U postgres -c "ALTER ROLE root WITH LOGIN;" - & "C:\program files\postgresql\10\bin\psql.exe" -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" - & "C:\program files\postgresql\10\bin\psql.exe" -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" - & "C:\program files\postgresql\10\bin\psql.exe" -U postgres -c "ALTER ROLE noaccess WITH LOGIN;" - & "C:\program files\postgresql\10\bin\psql.exe" -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" + $serviceName = Get-Service -Name postgresql* + Set-Service -InputObject $serviceName -StartupType Automatic + Start-Service -InputObject $serviceName + + createdb.exe -U postgres dbt + psql.exe -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';" + psql.exe -U postgres -c "ALTER ROLE root WITH LOGIN;" + psql.exe -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" + psql.exe -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" + psql.exe -U postgres -c "ALTER ROLE noaccess WITH LOGIN;" + psql.exe -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" displayName: Install postgresql and set up database - task: UsePythonVersion@0 From e2d70270740ace3a3979a0d4897d0735d3a7784d Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 10 Jun 2020 15:03:40 -0600 Subject: [PATCH 3/5] add a test --- .../test_custom_schema.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/test/integration/024_custom_schema_test/test_custom_schema.py b/test/integration/024_custom_schema_test/test_custom_schema.py index 065c2fb84d6..724ac5c45e9 100644 --- a/test/integration/024_custom_schema_test/test_custom_schema.py +++ b/test/integration/024_custom_schema_test/test_custom_schema.py @@ -66,17 +66,36 @@ def project_config(self): }, } + def _list_schemas(self): + with self.get_connection(): + return set(self.adapter.list_schemas(self.default_database)) + + def assert_schemas_created(self, expected): + assert self._list_schemas().intersection(expected) == expected + + def assert_schemas_not_created(self, expected): + assert not self._list_schemas().intersection(expected) + @use_profile('postgres') def test__postgres__custom_schema_with_prefix(self): + schema = self.unique_schema() + v1_schema = f"{schema}_dbt_test" + v2_schema = f"{schema}_custom" + xf_schema = f"{schema}_test" + new_schemas = {v1_schema, v2_schema, xf_schema} + + self.assert_schemas_not_created(new_schemas) + self.run_sql_file("seed.sql") + self.run_dbt(['ls']) + self.assert_schemas_not_created(new_schemas) + self.run_dbt(['compile']) + self.assert_schemas_not_created(new_schemas) + results = self.run_dbt() self.assertEqual(len(results), 3) - - schema = self.unique_schema() - v1_schema = "{}_dbt_test".format(schema) - v2_schema = "{}_custom".format(schema) - xf_schema = "{}_test".format(schema) + self.assert_schemas_created(new_schemas) self.assertTablesEqual("seed", "view_1", schema, v1_schema) self.assertTablesEqual("seed", "view_2", schema, v2_schema) From 2f36d7d259edd1cd24dd681df995b7602425efea Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 10 Jun 2020 15:08:46 -0600 Subject: [PATCH 4/5] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15bae98f2f1..bc11cf0e1bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## dbt 0.17.1 (Release TBD) + +### Fixes +- dbt compile and ls no longer create schemas if they don't already exist ([#2525](https://github.com/fishtown-analytics/dbt/issues/2525), [#2528](https://github.com/fishtown-analytics/dbt/pull/2528)) + + ## dbt 0.17.0 (June 08, 2020) ### Fixes From 947a00f6c94b62430eede15319a8d9ebc399e7a2 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 10 Jun 2020 15:11:31 -0600 Subject: [PATCH 5/5] =?UTF-8?q?Bump=20version:=200.17.0=20=E2=86=92=200.17?= =?UTF-8?q?.1a1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 3 ++- core/dbt/version.py | 2 +- core/setup.py | 2 +- plugins/bigquery/dbt/adapters/bigquery/__version__.py | 2 +- plugins/bigquery/setup.py | 2 +- plugins/postgres/dbt/adapters/postgres/__version__.py | 2 +- plugins/postgres/setup.py | 2 +- plugins/redshift/dbt/adapters/redshift/__version__.py | 2 +- plugins/redshift/setup.py | 2 +- plugins/snowflake/dbt/adapters/snowflake/__version__.py | 2 +- plugins/snowflake/setup.py | 2 +- setup.py | 2 +- 12 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7d4e77a4ba1..76045d8e247 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.17.0 +current_version = 0.17.1a1 parse = (?P\d+) \.(?P\d+) \.(?P\d+) @@ -41,3 +41,4 @@ first_value = 1 [bumpversion:file:plugins/snowflake/dbt/adapters/snowflake/__version__.py] [bumpversion:file:plugins/bigquery/dbt/adapters/bigquery/__version__.py] + diff --git a/core/dbt/version.py b/core/dbt/version.py index e58304522b9..c3bf7765592 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -96,5 +96,5 @@ def _get_dbt_plugins_info(): yield plugin_name, mod.version -__version__ = '0.17.0' +__version__ = '0.17.1a1' installed = get_installed_version() diff --git a/core/setup.py b/core/setup.py index 4e3d9f98a1a..b63daad5d0a 100644 --- a/core/setup.py +++ b/core/setup.py @@ -18,7 +18,7 @@ def read(fname): package_name = "dbt-core" -package_version = "0.17.0" +package_version = "0.17.1a1" description = """dbt (data build tool) is a command line tool that helps \ analysts and engineers transform data in their warehouse more effectively""" diff --git a/plugins/bigquery/dbt/adapters/bigquery/__version__.py b/plugins/bigquery/dbt/adapters/bigquery/__version__.py index 0631519830b..45490b1a275 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/__version__.py +++ b/plugins/bigquery/dbt/adapters/bigquery/__version__.py @@ -1 +1 @@ -version = '0.17.0' +version = '0.17.1a1' diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py index e4d76a5d273..409e2902f8e 100644 --- a/plugins/bigquery/setup.py +++ b/plugins/bigquery/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-bigquery" -package_version = "0.17.0" +package_version = "0.17.1a1" description = """The bigquery adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/postgres/dbt/adapters/postgres/__version__.py b/plugins/postgres/dbt/adapters/postgres/__version__.py index 0631519830b..45490b1a275 100644 --- a/plugins/postgres/dbt/adapters/postgres/__version__.py +++ b/plugins/postgres/dbt/adapters/postgres/__version__.py @@ -1 +1 @@ -version = '0.17.0' +version = '0.17.1a1' diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index f8230f2f5a9..7903817b386 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -35,7 +35,7 @@ def _dbt_psycopg2_name(): package_name = "dbt-postgres" -package_version = "0.17.0" +package_version = "0.17.1a1" description = """The postgres adpter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/redshift/dbt/adapters/redshift/__version__.py b/plugins/redshift/dbt/adapters/redshift/__version__.py index 0631519830b..45490b1a275 100644 --- a/plugins/redshift/dbt/adapters/redshift/__version__.py +++ b/plugins/redshift/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = '0.17.0' +version = '0.17.1a1' diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py index a35978eb95f..68caa022aac 100644 --- a/plugins/redshift/setup.py +++ b/plugins/redshift/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-redshift" -package_version = "0.17.0" +package_version = "0.17.1a1" description = """The redshift adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/snowflake/dbt/adapters/snowflake/__version__.py b/plugins/snowflake/dbt/adapters/snowflake/__version__.py index 0631519830b..45490b1a275 100644 --- a/plugins/snowflake/dbt/adapters/snowflake/__version__.py +++ b/plugins/snowflake/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = '0.17.0' +version = '0.17.1a1' diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py index 54c7c47c59f..357362ad4d2 100644 --- a/plugins/snowflake/setup.py +++ b/plugins/snowflake/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-snowflake" -package_version = "0.17.0" +package_version = "0.17.1a1" description = """The snowflake adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/setup.py b/setup.py index 010ecc1a539..96dd765d95b 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ package_name = "dbt" -package_version = "0.17.0" +package_version = "0.17.1a1" description = """With dbt, data analysts and engineers can build analytics \ the way engineers build applications."""