Merge pull request #2521 from azhard/bigquery-rename-relation

Added support for BigQuery relation renaming

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Added a "docs" field to macros, with a "show" subfield to allow for hiding macros from the documentation site ([#2430](https://github.com/fishtown-analytics/dbt/issues/2430))
 - Added intersection syntax for model selector ([#2167](https://github.com/fishtown-analytics/dbt/issues/2167), [#2417](https://github.com/fishtown-analytics/dbt/pull/2417))
 - Extends model selection syntax with at most n-th parent/children `dbt run --models 3+m1+2` ([#2052](https://github.com/fishtown-analytics/dbt/issues/2052), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485))
+- Added support for renaming BigQuery relations ([#2520](https://github.com/fishtown-analytics/dbt/issues/2520), [#2521](https://github.com/fishtown-analytics/dbt/pull/2521))
 
 ### Fixes
 - Fixed an error in create_adapter_plugins.py script when -dependency arg not passed ([#2507](https://github.com/fishtown-analytics/dbt/issues/2507), [#2508](https://github.com/fishtown-analytics/dbt/pull/2508))
@@ -17,7 +18,8 @@ Contributors:
  - [@raalsky](https://github.com/Raalsky) ([#2417](https://github.com/fishtown-analytics/dbt/pull/2417), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485))
  - [@alf-mindshift](https://github.com/alf-mindshift) ([#2431](https://github.com/fishtown-analytics/dbt/pull/2431))
  - [@scarrucciu](https://github.com/scarrucciu) ([#2508](https://github.com/fishtown-analytics/dbt/pull/2508))
-- [@southpolemonkey](https://github.com/southpolemonkey)([#2511](https://github.com/fishtown-analytics/dbt/issues/2511))
+ - [@southpolemonkey](https://github.com/southpolemonkey)([#2511](https://github.com/fishtown-analytics/dbt/issues/2511))
+ - [@azhard](https://github.com/azhard) ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521)
 
 ## dbt 0.17.0 (June 08, 2020)
 

Makefile

@@ -15,7 +15,7 @@ test-unit:
 
 test-integration:
 	@echo "Integration test run starting..."
-	@time docker-compose run test tox -e integration-postgres-py36integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36
+	@time docker-compose run test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36
 
 test-quick:
 	@echo "Integration test run starting..."

plugins/bigquery/dbt/adapters/bigquery/impl.py

@@ -143,9 +143,30 @@ def truncate_relation(self, relation: BigQueryRelation) -> None:
     def rename_relation(
         self, from_relation: BigQueryRelation, to_relation: BigQueryRelation
     ) -> None:
-        raise dbt.exceptions.NotImplementedException(
-            '`rename_relation` is not implemented for this adapter!'
-        )
+
+        conn = self.connections.get_thread_connection()
+        client = conn.handle
+
+        from_table_ref = self.connections.table_ref(from_relation.database,
+                                                    from_relation.schema,
+                                                    from_relation.identifier,
+                                                    conn)
+        from_table = client.get_table(from_table_ref)
+        if from_table.table_type == "VIEW" or \
+                from_relation.type == RelationType.View or \
+                to_relation.type == RelationType.View:
+            raise dbt.exceptions.RuntimeException(
+                'Renaming of views is not currently supported in BigQuery'
+            )
+
+        to_table_ref = self.connections.table_ref(to_relation.database,
+                                                  to_relation.schema,
+                                                  to_relation.identifier,
+                                                  conn)
+
+        self.cache_renamed(from_relation, to_relation)
+        client.copy_table(from_table_ref, to_table_ref)
+        client.delete_table(from_table_ref)
 
     @available
     def list_schemas(self, database: str) -> List[str]:

plugins/bigquery/dbt/include/bigquery/macros/adapters.sql

@@ -123,3 +123,7 @@
 {% macro bigquery__alter_column_comment(relation, column_dict) -%}
   {% do adapter.update_column_descriptions(relation, column_dict) %}
 {% endmacro %}
+
+{% macro bigquery__rename_relation(from_relation, to_relation) -%}
+  {% do adapter.rename_relation(from_relation, to_relation) %}
+{% endmacro %}

test.env.sample

@@ -1,4 +1,5 @@
 DBT_INVOCATION_ENV=development
+
 SNOWFLAKE_TEST_ACCOUNT=
 SNOWFLAKE_TEST_USER=
 SNOWFLAKE_TEST_PASSWORD=
@@ -11,16 +12,8 @@ SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN=
 SNOWFLAKE_TEST_OAUTH_CLIENT_ID=
 SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET=
 
-BIGQUERY_TYPE=
-BIGQUERY_PROJECT_ID=
-BIGQUERY_PRIVATE_KEY_ID=
-BIGQUERY_PRIVATE_KEY=
-BIGQUERY_CLIENT_EMAIL=
-BIGQUERY_CLIENT_ID=
-BIGQUERY_AUTH_URI=
-BIGQUERY_TOKEN_URI=
-BIGQUERY_AUTH_PROVIDER_X509_CERT_URL=
-BIGQUERY_CLIENT_X509_CERT_URL=
+BIGQUERY_SERVICE_ACCOUNT_JSON=
+BIGQUERY_TEST_ALT_DATABASE=
 
 REDSHIFT_TEST_HOST=
 REDSHIFT_TEST_USER=

test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql

@@ -0,0 +1 @@
+select * from {{ source('test_source', 'renamed_seed') }}

test/integration/054_adapter_methods_test/bigquery-models/sources.yml

@@ -0,0 +1,6 @@
+version: 2
+sources:
+  - name: test_source
+    schema: "{{ target.schema }}"
+    tables:
+      - name: renamed_seed

test/integration/054_adapter_methods_test/macros/rename_named_relation.sql

@@ -0,0 +1,6 @@
+-- Macro to rename a relation
+{% macro rename_named_relation(from_name, to_name) %}
+{%- set from_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=from_name, type='table') -%}
+{%- set to_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=to_name, type='table') -%}
+{% do adapter.rename_relation(from_relation, to_relation) %}
+{% endmacro %}

test/integration/054_adapter_methods_test/seed_bq.sql

@@ -0,0 +1,32 @@
+create table {database}.{schema}.seed (
+	id INT64,
+	first_name STRING,
+	last_name STRING,
+	email STRING,
+	gender STRING,
+	ip_address STRING,
+	updated_at TIMESTAMP
+);
+
+-- seed inserts
+insert {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values
+(1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'),
+(2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'),
+(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'),
+(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'),
+(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'),
+(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'),
+(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'),
+(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'),
+(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'),
+(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'),
+(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'),
+(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'),
+(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'),
+(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'),
+(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'),
+(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'),
+(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'),
+(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'),
+(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'),
+(20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19');

test/integration/054_adapter_methods_test/test_adapter_methods.py

@@ -1,4 +1,5 @@
 from test.integration.base import DBTIntegrationTest, use_profile
+import yaml
 
 
 class TestBaseCaching(DBTIntegrationTest):
@@ -40,3 +41,32 @@ def test_bigquery_adapter_methods(self):
         self.run_dbt(['compile'])  # trigger any compile-time issues
         self.run_dbt()
         self.assertTablesEqual('model', 'expected')
+
+
+class TestRenameRelation(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "rename_relation_054"
+
+    @property
+    def models(self):
+        return 'bigquery-models'
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'source-paths': ['models']
+        }
+
+    @use_profile('bigquery')
+    def test_bigquery_adapter_methods(self):
+        self.run_dbt(['compile'])  # trigger any compile-time issues
+        self.run_sql_file("seed_bq.sql")
+        self.run_dbt(['seed'])
+        rename_relation_args = yaml.safe_dump({
+            'from_name': 'seed',
+            'to_name': 'renamed_seed',
+        })
+        self.run_dbt(['run-operation', 'rename_named_relation', '--args', rename_relation_args])
+        self.run_dbt()