From 0f6622fab671867fadf099aaf27eb0fbf79b4f70 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Tue, 9 Jun 2020 11:17:17 -0400 Subject: [PATCH 1/6] Added support for BigQuery relation renaming --- CHANGELOG.md | 4 +++- .../bigquery/dbt/adapters/bigquery/impl.py | 24 ++++++++++++++++--- .../dbt/include/bigquery/macros/adapters.sql | 4 ++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef79df452af..6d9691b229a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Added a "docs" field to macros, with a "show" subfield to allow for hiding macros from the documentation site ([#2430](https://github.com/fishtown-analytics/dbt/issues/2430)) - Added intersection syntax for model selector ([#2167](https://github.com/fishtown-analytics/dbt/issues/2167), [#2417](https://github.com/fishtown-analytics/dbt/pull/2417)) - Extends model selection syntax with at most n-th parent/children `dbt run --models 3+m1+2` ([#2052](https://github.com/fishtown-analytics/dbt/issues/2052), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) +- Added support for renaming BigQuery relations ([#2520](https://github.com/fishtown-analytics/dbt/issues/2520), [#2521](https://github.com/fishtown-analytics/dbt/pull/2521)) ### Fixes - Fixed an error in create_adapter_plugins.py script when -dependency arg not passed ([#2507](https://github.com/fishtown-analytics/dbt/issues/2507), [#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) @@ -14,7 +15,8 @@ Contributors: - [@raalsky](https://github.com/Raalsky) ([#2417](https://github.com/fishtown-analytics/dbt/pull/2417), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) - [@alf-mindshift](https://github.com/alf-mindshift) ([#2431](https://github.com/fishtown-analytics/dbt/pull/2431)) - [@scarrucciu](https://github.com/scarrucciu) ([#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) -- [@southpolemonkey](https://github.com/southpolemonkey)([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) + - [@southpolemonkey](https://github.com/southpolemonkey)([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) + - [@azhard](https://github.com/azhard) ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521) ## dbt 0.17.0 (June 08, 2020) diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index caafcb0fbfb..111189234f5 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -143,9 +143,27 @@ def truncate_relation(self, relation: BigQueryRelation) -> None: def rename_relation( self, from_relation: BigQueryRelation, to_relation: BigQueryRelation ) -> None: - raise dbt.exceptions.NotImplementedException( - '`rename_relation` is not implemented for this adapter!' - ) + self.cache_renamed(from_relation, to_relation) + + conn = self.connections.get_thread_connection() + client = conn.handle + + from_table_ref = self.connections.table_ref(from_relation.database, + from_relation.schema, + from_relation.identifier, + conn) + from_table = client.get_table(from_table_ref) + if from_table.table_type == "VIEW": + raise dbt.exceptions.RuntimeException( + 'Renaming of views is not currently supported in BigQuery' + ) + + to_table_ref = self.connections.table_ref(to_relation.database, + to_relation.schema, + to_relation.identifier, + conn) + client.copy_table(from_table_ref, to_table_ref) + client.delete_table(from_table_ref) @available def list_schemas(self, database: str) -> List[str]: diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql index d6365a27653..1a06a545761 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql @@ -123,3 +123,7 @@ {% macro bigquery__alter_column_comment(relation, column_dict) -%} {% do adapter.update_column_descriptions(relation, column_dict) %} {% endmacro %} + +{% macro bigquery__rename_relation(from_relation, to_relation) -%} + {% do adapter.rename_relation(from_relation, to_relation) %} +{% endmacro %} From 75cc3e40a5f231071913b2b15a835dcda1031e9b Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Tue, 9 Jun 2020 14:43:59 -0400 Subject: [PATCH 2/6] Addressed PR comments --- .../bigquery/dbt/adapters/bigquery/impl.py | 7 +- .../macros/rename_named_relation.sql | 6 + .../models/renamed_model.sql | 1 + .../models/sources.yml | 6 + .../054_adapter_methods_test/seed.sql | 111 ++++++++++++++++++ .../test_adapter_methods.py | 73 ++++++++---- 6 files changed, 180 insertions(+), 24 deletions(-) create mode 100644 test/integration/054_adapter_methods_test/macros/rename_named_relation.sql create mode 100644 test/integration/054_adapter_methods_test/models/renamed_model.sql create mode 100644 test/integration/054_adapter_methods_test/models/sources.yml create mode 100644 test/integration/054_adapter_methods_test/seed.sql diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index 111189234f5..762e579d634 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -143,7 +143,6 @@ def truncate_relation(self, relation: BigQueryRelation) -> None: def rename_relation( self, from_relation: BigQueryRelation, to_relation: BigQueryRelation ) -> None: - self.cache_renamed(from_relation, to_relation) conn = self.connections.get_thread_connection() client = conn.handle @@ -153,7 +152,9 @@ def rename_relation( from_relation.identifier, conn) from_table = client.get_table(from_table_ref) - if from_table.table_type == "VIEW": + if from_table.table_type == "VIEW" or \ + from_relation.type == RelationType.View or \ + to_relation.type == RelationType.View: raise dbt.exceptions.RuntimeException( 'Renaming of views is not currently supported in BigQuery' ) @@ -162,6 +163,8 @@ def rename_relation( to_relation.schema, to_relation.identifier, conn) + + self.cache_renamed(from_relation, to_relation) client.copy_table(from_table_ref, to_table_ref) client.delete_table(from_table_ref) diff --git a/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql b/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql new file mode 100644 index 00000000000..c7d92d4abd4 --- /dev/null +++ b/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql @@ -0,0 +1,6 @@ +-- Macro to rename a relation +{% macro rename_named_relation(from_name, to_name) %} +{%- set from_relation = adapter.get_relation(database=target.database, schema=target.schema, identifier=from_name) -%} +{%- set to_relation = adapter.get_relation(database=target.database, schema=target.schema, identifier=to_name) -%} +{% do adapter.rename_relation(from_relation, to_relation) %} +{% endmacro %} \ No newline at end of file diff --git a/test/integration/054_adapter_methods_test/models/renamed_model.sql b/test/integration/054_adapter_methods_test/models/renamed_model.sql new file mode 100644 index 00000000000..3c030d920ec --- /dev/null +++ b/test/integration/054_adapter_methods_test/models/renamed_model.sql @@ -0,0 +1 @@ +select * from {{ source('test_source', 'renamed_seed') }} \ No newline at end of file diff --git a/test/integration/054_adapter_methods_test/models/sources.yml b/test/integration/054_adapter_methods_test/models/sources.yml new file mode 100644 index 00000000000..cb74cde9c05 --- /dev/null +++ b/test/integration/054_adapter_methods_test/models/sources.yml @@ -0,0 +1,6 @@ +version: 2 +sources: + - name: test_source + schema: "{{ target.schema }}" + tables: + - name: renamed_seed diff --git a/test/integration/054_adapter_methods_test/seed.sql b/test/integration/054_adapter_methods_test/seed.sql new file mode 100644 index 00000000000..3c618ccab3a --- /dev/null +++ b/test/integration/054_adapter_methods_test/seed.sql @@ -0,0 +1,111 @@ +create table {schema}.seed ( + id BIGSERIAL PRIMARY KEY, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(50), + gender VARCHAR(50), + ip_address VARCHAR(20) +); + + +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values +('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), +('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), +('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), +('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'), +('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'), +('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'), +('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'), +('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'), +('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'), +('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'), +('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'), +('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'), +('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'), +('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'), +('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'), +('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'), +('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'), +('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'), +('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'), +('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'), +('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'), +('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'), +('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'), +('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'), +('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'), +('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'), +('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'), +('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'), +('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'), +('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'), +('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'), +('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'), +('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'), +('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'), +('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'), +('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'), +('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'), +('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'), +('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'), +('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'), +('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'), +('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'), +('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'), +('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'), +('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'), +('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'), +('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'), +('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'), +('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'), +('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'), +('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'), +('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'), +('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'), +('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'), +('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'), +('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'), +('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'), +('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'), +('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'), +('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'), +('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'), +('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'), +('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'), +('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'), +('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'), +('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'), +('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'), +('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'), +('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'), +('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'), +('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'), +('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'), +('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'), +('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'), +('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'), +('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'), +('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'), +('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'), +('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'), +('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'), +('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'), +('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'), +('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'), +('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'), +('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'), +('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'), +('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'), +('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'), +('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'), +('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'), +('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'), +('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'), +('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'), +('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'), +('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'), +('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'), +('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'), +('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'), +('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'), +('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index 72d17fb0a09..9e89b184a20 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -1,10 +1,52 @@ from test.integration.base import DBTIntegrationTest, use_profile +import yaml -class TestBaseCaching(DBTIntegrationTest): +# class TestBaseCaching(DBTIntegrationTest): +# @property +# def schema(self): +# return "caching_038" +# +# @property +# def models(self): +# return "models" +# +# @property +# def project_config(self): +# return { +# 'config-version': 2, +# 'test-paths': ['tests'] +# } +# +# @use_profile('postgres') +# def test_postgres_adapter_methods(self): +# self.run_dbt(['compile']) # trigger any compile-time issues +# self.run_dbt() +# self.assertTablesEqual('model', 'expected') +# +# @use_profile('redshift') +# def test_redshift_adapter_methods(self): +# self.run_dbt(['compile']) # trigger any compile-time issues +# self.run_dbt() +# self.assertTablesEqual('model', 'expected') +# +# @use_profile('snowflake') +# def test_snowflake_adapter_methods(self): +# self.run_dbt(['compile']) # trigger any compile-time issues +# self.run_dbt() +# self.assertTablesEqual('MODEL', 'EXPECTED') +# +# @use_profile('bigquery') +# def test_bigquery_adapter_methods(self): +# self.run_dbt(['compile']) # trigger any compile-time issues +# self.run_dbt() +# self.assertTablesEqual('model', 'expected') + + +class TestRenameRelation(DBTIntegrationTest): @property def schema(self): - return "caching_038" + return "rename_relation_054" @property def models(self): @@ -14,29 +56,16 @@ def models(self): def project_config(self): return { 'config-version': 2, - 'test-paths': ['tests'] + 'source-paths': ['models'] } - @use_profile('postgres') - def test_postgres_adapter_methods(self): - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_dbt() - self.assertTablesEqual('model', 'expected') - - @use_profile('redshift') - def test_redshift_adapter_methods(self): - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_dbt() - self.assertTablesEqual('model', 'expected') - - @use_profile('snowflake') - def test_snowflake_adapter_methods(self): - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_dbt() - self.assertTablesEqual('MODEL', 'EXPECTED') - @use_profile('bigquery') def test_bigquery_adapter_methods(self): self.run_dbt(['compile']) # trigger any compile-time issues + self.run_dbt(['seed']) + rename_relation_args = yaml.safe_dump({ + 'from_name': 'seed', + 'to_name': 'renamed_seed', + }) + self.run_dbt(['run-operation', 'rename_named_relation', '--args', rename_relation_args]) self.run_dbt() - self.assertTablesEqual('model', 'expected') From 7aa09e393c705a89d841386ec386e00b882c4301 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Tue, 9 Jun 2020 14:50:08 -0400 Subject: [PATCH 3/6] Undo accidental commented out test --- .../test_adapter_methods.py | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index 9e89b184a20..89a4311af5b 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -2,45 +2,45 @@ import yaml -# class TestBaseCaching(DBTIntegrationTest): -# @property -# def schema(self): -# return "caching_038" -# -# @property -# def models(self): -# return "models" -# -# @property -# def project_config(self): -# return { -# 'config-version': 2, -# 'test-paths': ['tests'] -# } -# -# @use_profile('postgres') -# def test_postgres_adapter_methods(self): -# self.run_dbt(['compile']) # trigger any compile-time issues -# self.run_dbt() -# self.assertTablesEqual('model', 'expected') -# -# @use_profile('redshift') -# def test_redshift_adapter_methods(self): -# self.run_dbt(['compile']) # trigger any compile-time issues -# self.run_dbt() -# self.assertTablesEqual('model', 'expected') -# -# @use_profile('snowflake') -# def test_snowflake_adapter_methods(self): -# self.run_dbt(['compile']) # trigger any compile-time issues -# self.run_dbt() -# self.assertTablesEqual('MODEL', 'EXPECTED') -# -# @use_profile('bigquery') -# def test_bigquery_adapter_methods(self): -# self.run_dbt(['compile']) # trigger any compile-time issues -# self.run_dbt() -# self.assertTablesEqual('model', 'expected') +class TestBaseCaching(DBTIntegrationTest): + @property + def schema(self): + return "caching_038" + + @property + def models(self): + return "models" + + @property + def project_config(self): + return { + 'config-version': 2, + 'test-paths': ['tests'] + } + + @use_profile('postgres') + def test_postgres_adapter_methods(self): + self.run_dbt(['compile']) # trigger any compile-time issues + self.run_dbt() + self.assertTablesEqual('model', 'expected') + + @use_profile('redshift') + def test_redshift_adapter_methods(self): + self.run_dbt(['compile']) # trigger any compile-time issues + self.run_dbt() + self.assertTablesEqual('model', 'expected') + + @use_profile('snowflake') + def test_snowflake_adapter_methods(self): + self.run_dbt(['compile']) # trigger any compile-time issues + self.run_dbt() + self.assertTablesEqual('MODEL', 'EXPECTED') + + @use_profile('bigquery') + def test_bigquery_adapter_methods(self): + self.run_dbt(['compile']) # trigger any compile-time issues + self.run_dbt() + self.assertTablesEqual('model', 'expected') class TestRenameRelation(DBTIntegrationTest): From 107170163a46cec5d78ddab6b26b1880fdd2b81e Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Tue, 9 Jun 2020 15:00:29 -0400 Subject: [PATCH 4/6] Fixed relation creation --- .../054_adapter_methods_test/macros/rename_named_relation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql b/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql index c7d92d4abd4..253e1e0ad8a 100644 --- a/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql +++ b/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql @@ -1,6 +1,6 @@ -- Macro to rename a relation {% macro rename_named_relation(from_name, to_name) %} -{%- set from_relation = adapter.get_relation(database=target.database, schema=target.schema, identifier=from_name) -%} -{%- set to_relation = adapter.get_relation(database=target.database, schema=target.schema, identifier=to_name) -%} +{%- set from_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=from_name, type='table') -%} +{%- set to_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=to_name, type='table') -%} {% do adapter.rename_relation(from_relation, to_relation) %} {% endmacro %} \ No newline at end of file From ddb6d791e86b92031b99479088f8f854ebba8fd8 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Tue, 9 Jun 2020 16:28:00 -0400 Subject: [PATCH 5/6] Fixed CI errors --- Makefile | 2 +- test.env.sample | 13 +- .../renamed_model.sql | 0 .../{models => bigquery-models}/sources.yml | 0 .../054_adapter_methods_test/seed.sql | 139 ++++-------------- .../test_adapter_methods.py | 3 +- 6 files changed, 36 insertions(+), 121 deletions(-) rename test/integration/054_adapter_methods_test/{models => bigquery-models}/renamed_model.sql (100%) rename test/integration/054_adapter_methods_test/{models => bigquery-models}/sources.yml (100%) diff --git a/Makefile b/Makefile index 9219810d4ac..5da89e89584 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test-unit: test-integration: @echo "Integration test run starting..." - @time docker-compose run test tox -e integration-postgres-py36integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36 + @time docker-compose run test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36 test-quick: @echo "Integration test run starting..." diff --git a/test.env.sample b/test.env.sample index 9f87a3d9461..2ddf896f8f8 100644 --- a/test.env.sample +++ b/test.env.sample @@ -1,4 +1,5 @@ DBT_INVOCATION_ENV=development + SNOWFLAKE_TEST_ACCOUNT= SNOWFLAKE_TEST_USER= SNOWFLAKE_TEST_PASSWORD= @@ -11,16 +12,8 @@ SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN= SNOWFLAKE_TEST_OAUTH_CLIENT_ID= SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET= -BIGQUERY_TYPE= -BIGQUERY_PROJECT_ID= -BIGQUERY_PRIVATE_KEY_ID= -BIGQUERY_PRIVATE_KEY= -BIGQUERY_CLIENT_EMAIL= -BIGQUERY_CLIENT_ID= -BIGQUERY_AUTH_URI= -BIGQUERY_TOKEN_URI= -BIGQUERY_AUTH_PROVIDER_X509_CERT_URL= -BIGQUERY_CLIENT_X509_CERT_URL= +BIGQUERY_SERVICE_ACCOUNT_JSON= +BIGQUERY_TEST_ALT_DATABASE= REDSHIFT_TEST_HOST= REDSHIFT_TEST_USER= diff --git a/test/integration/054_adapter_methods_test/models/renamed_model.sql b/test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql similarity index 100% rename from test/integration/054_adapter_methods_test/models/renamed_model.sql rename to test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql diff --git a/test/integration/054_adapter_methods_test/models/sources.yml b/test/integration/054_adapter_methods_test/bigquery-models/sources.yml similarity index 100% rename from test/integration/054_adapter_methods_test/models/sources.yml rename to test/integration/054_adapter_methods_test/bigquery-models/sources.yml diff --git a/test/integration/054_adapter_methods_test/seed.sql b/test/integration/054_adapter_methods_test/seed.sql index 3c618ccab3a..71a9a78c665 100644 --- a/test/integration/054_adapter_methods_test/seed.sql +++ b/test/integration/054_adapter_methods_test/seed.sql @@ -1,111 +1,32 @@ -create table {schema}.seed ( - id BIGSERIAL PRIMARY KEY, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20) +create table {database}.{schema}.seed ( + id INT64, + first_name STRING, + last_name STRING, + email STRING, + gender STRING, + ip_address STRING, + updated_at TIMESTAMP ); - -insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values -('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), -('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), -('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), -('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'), -('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'), -('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'), -('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'), -('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'), -('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'), -('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'), -('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'), -('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'), -('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'), -('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'), -('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'), -('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'), -('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'), -('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'), -('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'), -('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'), -('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'), -('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'), -('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'), -('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'), -('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'), -('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'), -('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'), -('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'), -('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'), -('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'), -('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'), -('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'), -('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'), -('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'), -('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'), -('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'), -('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'), -('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'), -('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'), -('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'), -('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'), -('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'), -('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'), -('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'), -('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'), -('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'), -('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'), -('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'), -('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'), -('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'), -('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'), -('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'), -('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'), -('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'), -('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'), -('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'), -('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'), -('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'), -('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'), -('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'), -('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'), -('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'), -('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'), -('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'), -('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'), -('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'), -('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'), -('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'), -('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'), -('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'), -('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'), -('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'), -('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'), -('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'), -('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'), -('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'), -('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'), -('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'), -('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'), -('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'), -('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'), -('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'), -('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'), -('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'), -('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'), -('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'), -('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'), -('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'), -('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'), -('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'), -('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'), -('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'), -('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'), -('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'), -('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'), -('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'), -('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'), -('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'), -('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'), -('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); +-- seed inserts +insert {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values +(1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), +(2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), +(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), +(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), +(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), +(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), +(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), +(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), +(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), +(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), +(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), +(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), +(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), +(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), +(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), +(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), +(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), +(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), +(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), +(20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index 89a4311af5b..c431550433b 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -50,7 +50,7 @@ def schema(self): @property def models(self): - return "models" + return 'bigquery-models' @property def project_config(self): @@ -62,6 +62,7 @@ def project_config(self): @use_profile('bigquery') def test_bigquery_adapter_methods(self): self.run_dbt(['compile']) # trigger any compile-time issues + self.run_sql_file("seed_bq.sql") self.run_dbt(['seed']) rename_relation_args = yaml.safe_dump({ 'from_name': 'seed', From daed251d0d750f02e5e27cb3c3b66be6f8456f43 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Tue, 9 Jun 2020 17:04:42 -0400 Subject: [PATCH 6/6] Fix seed file name --- .../054_adapter_methods_test/{seed.sql => seed_bq.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/integration/054_adapter_methods_test/{seed.sql => seed_bq.sql} (100%) diff --git a/test/integration/054_adapter_methods_test/seed.sql b/test/integration/054_adapter_methods_test/seed_bq.sql similarity index 100% rename from test/integration/054_adapter_methods_test/seed.sql rename to test/integration/054_adapter_methods_test/seed_bq.sql