Merge pull request #1 from fishtown-analytics/dev/0.18.1

Dev/0.18.1

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.18.1b2
+current_version = 0.18.1b3
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)

CHANGELOG.md

@@ -1,10 +1,22 @@
 ## dbt 0.18.1 (Release TBD)
 
+## dbt 0.18.1b3 (September 25, 2020)
+
+
 ### Feature
 - Added 'Last Modified' stat in snowflake catalog macro. Now should be available in docs. ([#2728](https://github.com/fishtown-analytics/dbt/issues/2728))
 
+### Fixes
+- `dbt compile` and `dbt run` failed with `KeyError: 'endpoint_resolver'` when threads > 1 and `method: iam` had been specified in the profiles.yaml ([#2756](https://github.com/fishtown-analytics/dbt/issues/2756), [#2766](https://github.com/fishtown-analytics/dbt/pull/2766))
+- Fix Redshift adapter to include columns from external tables when using the get_columns_in_relation macro ([#2753](https://github.com/fishtown-analytics/dbt/issues/2753), [#2754](https://github.com/fishtown-analytics/dbt/pull/2754))
+
+### Under the hood
+- Require extra `snowflake-connector-python[secure-local-storage]` on all dbt-snowflake installations ([#2779](https://github.com/fishtown-analytics/dbt/issues/2779), [#2789](https://github.com/fishtown-analytics/dbt/pull/2789))
+
 Contributors:
 - [@Mr-Nobody99](https://github.com/Mr-Nobody99) ([#2732](https://github.com/fishtown-analytics/dbt/pull/2732))
+- [@jweibel22](https://github.com/jweibel22) ([#2766](https://github.com/fishtown-analytics/dbt/pull/2766))
+- [@aiguofer](https://github.com/aiguofer) ([#2754](https://github.com/fishtown-analytics/dbt/pull/2754))
 
 ## dbt 0.18.1b2 (September 22, 2020)
 

core/dbt/version.py

@@ -96,5 +96,5 @@ def _get_dbt_plugins_info():
         yield plugin_name, mod.version
 
 
-__version__ = '0.18.1b2'
+__version__ = '0.18.1b3'
 installed = get_installed_version()

core/setup.py

@@ -24,7 +24,7 @@ def read(fname):
 
 
 package_name = "dbt-core"
-package_version = "0.18.1b2"
+package_version = "0.18.1b3"
 description = """dbt (data build tool) is a command line tool that helps \
 analysts and engineers transform data in their warehouse more effectively"""
 

docker/requirements/requirements.0.18.1b3.txt

@@ -0,0 +1,69 @@
+agate==1.6.1
+asn1crypto==1.4.0
+attrs==20.2.0
+azure-common==1.1.25
+azure-core==1.8.1
+azure-storage-blob==12.5.0
+Babel==2.8.0
+boto3==1.11.17
+botocore==1.14.17
+cachetools==4.1.1
+certifi==2020.6.20
+cffi==1.14.3
+chardet==3.0.4
+colorama==0.4.3
+cryptography==2.9.2
+decorator==4.4.2
+docutils==0.15.2
+google-api-core==1.16.0
+google-auth==1.21.3
+google-cloud-bigquery==1.25.0
+google-cloud-core==1.3.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.6.0
+hologram==0.0.10
+idna==2.9
+importlib-metadata==2.0.0
+isodate==0.6.0
+jeepney==0.4.3
+Jinja2==2.11.2
+jmespath==0.10.0
+json-rpc==1.13.0
+jsonschema==3.1.1
+keyring==21.4.0
+leather==0.3.3
+Logbook==1.5.3
+MarkupSafe==1.1.1
+minimal-snowplow-tracker==0.0.2
+msrest==0.6.19
+networkx==2.5
+oauthlib==3.1.0
+oscrypto==1.2.1
+parsedatetime==2.6
+protobuf==3.11.3
+psycopg2-binary==2.8.6
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodomex==3.9.8
+PyJWT==1.7.1
+pyOpenSSL==19.1.0
+pyrsistent==0.17.3
+python-dateutil==2.8.1
+python-slugify==4.0.1
+pytimeparse==1.1.8
+pytz==2020.1
+PyYAML==5.3.1
+requests==2.23.0
+requests-oauthlib==1.3.0
+rsa==4.6
+s3transfer==0.3.3
+SecretStorage==3.1.2
+six==1.15.0
+snowflake-connector-python==2.2.10
+sqlparse==0.3.1
+text-unidecode==1.3
+typing-extensions==3.7.4.3
+urllib3==1.25.10
+Werkzeug==0.16.1
+zipp==3.2.0

plugins/bigquery/dbt/adapters/bigquery/__version__.py

@@ -1 +1 @@
-version = '0.18.1b2'
+version = '0.18.1b3'

plugins/bigquery/setup.py

@@ -20,7 +20,7 @@
 
 
 package_name = "dbt-bigquery"
-package_version = "0.18.1b2"
+package_version = "0.18.1b3"
 description = """The bigquery adapter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

plugins/postgres/dbt/adapters/postgres/__version__.py

@@ -1 +1 @@
-version = '0.18.1b2'
+version = '0.18.1b3'

plugins/postgres/setup.py

@@ -41,7 +41,7 @@ def _dbt_psycopg2_name():
 
 
 package_name = "dbt-postgres"
-package_version = "0.18.1b2"
+package_version = "0.18.1b3"
 description = """The postgres adpter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

plugins/redshift/dbt/adapters/redshift/__version__.py

@@ -1 +1 @@
-version = '0.18.1b2'
+version = '0.18.1b3'

plugins/redshift/dbt/adapters/redshift/connections.py

@@ -99,7 +99,8 @@ def fetch_cluster_credentials(cls, db_user, db_name, cluster_id,
         must already exist in the database, or else an error will occur"""
 
         if iam_profile is None:
-            boto_client = boto3.client('redshift')
+            session = boto3.Session()
+            boto_client = session.client("redshift")
         else:
             logger.debug("Connecting to Redshift using 'IAM'" +
                          f"with profile {iam_profile}")

plugins/redshift/dbt/include/redshift/macros/adapters.sql

@@ -129,10 +129,51 @@
       where view_name = '{{ relation.identifier }}'
     ),
 
+    external_views as (
+      select
+        columnnum,
+        schemaname,
+        columnname,
+        case
+          when external_type ilike 'character varying%' or external_type ilike 'varchar%'
+          then 'character varying'
+          when external_type ilike 'numeric%' then 'numeric'
+          else external_type
+        end as external_type,
+        case
+          when external_type like 'character%' or external_type like 'varchar%'
+          then nullif(
+            REGEXP_SUBSTR(external_type, '[0-9]+'),
+            '')::int
+          else null
+        end as character_maximum_length,
+        case
+          when external_type like 'numeric%'
+          then nullif(
+            SPLIT_PART(REGEXP_SUBSTR(external_type, '[0-9,]+'), ',', 1),
+            '')::int
+          else null
+        end as numeric_precision,
+        case
+          when external_type like 'numeric%'
+          then nullif(
+            SPLIT_PART(REGEXP_SUBSTR(external_type, '[0-9,]+'), ',', 2),
+            '')::int
+          else null
+        end as numeric_scale
+      from
+        pg_catalog.svv_external_columns
+      where
+        tablename = '{{ relation.identifier }}'
+
+    ),
+
     unioned as (
       select * from bound_views
       union all
       select * from unbound_views
+      union all
+      select * from external_views
     )
 
     select

plugins/redshift/setup.py

@@ -20,7 +20,7 @@
 
 
 package_name = "dbt-redshift"
-package_version = "0.18.1b2"
+package_version = "0.18.1b3"
 description = """The redshift adapter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

plugins/snowflake/dbt/adapters/snowflake/__version__.py

@@ -1 +1 @@
-version = '0.18.1b2'
+version = '0.18.1b3'

plugins/snowflake/setup.py

@@ -20,7 +20,7 @@
 
 
 package_name = "dbt-snowflake"
-package_version = "0.18.1b2"
+package_version = "0.18.1b3"
 description = """The snowflake adapter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))
@@ -47,7 +47,7 @@
     },
     install_requires=[
         'dbt-core=={}'.format(package_version),
-        'snowflake-connector-python==2.2.10',
+        'snowflake-connector-python[secure-local-storage]==2.2.10',
         'azure-common<2.0.0',
         'azure-storage-blob>=12.0.0,<13.0.0',
         'urllib3>=1.20,<1.26.0',

setup.py

@@ -24,7 +24,7 @@
 
 
 package_name = "dbt"
-package_version = "0.18.1b2"
+package_version = "0.18.1b3"
 description = """With dbt, data analysts and engineers can build analytics \
 the way engineers build applications."""
 

test/unit/test_redshift_adapter.py

@@ -1,6 +1,9 @@
 import unittest
 from unittest import mock
+from unittest.mock import Mock
+
 import agate
+import boto3
 
 import dbt.adapters  # noqa
 import dbt.flags as flags
@@ -128,6 +131,23 @@ def test_invalid_iam_no_cluster_id(self):
 
         self.assertTrue("'cluster_id' must be provided" in context.exception.msg)
 
+    def test_default_session_is_not_used_when_iam_used(self):
+        boto3.DEFAULT_SESSION = Mock()
+        self.config.credentials = self.config.credentials.replace(method='iam')
+        self.config.credentials.cluster_id = 'clusterid'
+        with mock.patch('dbt.adapters.redshift.connections.boto3.Session'):
+            RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials)
+            self.assertEquals(boto3.DEFAULT_SESSION.client.call_count, 0,
+                              "The redshift client should not be created using the default session because the session object is not thread-safe")
+
+    def test_default_session_is_not_used_when_iam_not_used(self):
+        boto3.DEFAULT_SESSION = Mock()
+        self.config.credentials = self.config.credentials.replace(method=None)
+        with mock.patch('dbt.adapters.redshift.connections.boto3.Session'):
+            RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials)
+            self.assertEquals(boto3.DEFAULT_SESSION.client.call_count, 0,
+                              "The redshift client should not be created using the default session because the session object is not thread-safe")
+
     def test_cancel_open_connections_empty(self):
         self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0)