All tests passing on BQ

core/dbt/context/base.py

@@ -1,6 +1,6 @@
 import json
 import os
-from typing import Any, Dict, NoReturn, Optional, Mapping, Iterable, Set
+from typing import Any, Dict, NoReturn, Optional, Mapping, Iterable, Set, List
 
 from dbt import flags
 from dbt import tracking
@@ -659,48 +659,27 @@ def print(msg: str) -> str:
 
     @contextmember
     @staticmethod
-    def diff_of_two_dicts(dict_a, dict_b):
+    def diff_of_two_dicts(
+        dict_a: Dict[str, List[str]], dict_b: Dict[str, List[str]]
+    ) -> Dict[str, List[str]]:
         """
-        Given two dictionaries:
-            dict_a: {'key_x': ['value_1', 'value_2'], 'key_y': ['value_3']}
-            dict_b: {'key_x': ['value_1'], 'key_z': ['value_4']}
-        Return the same dictionary representation of dict_a MINUS dict_b
-
-        All keys returned will be lower case.
-        """
-
-        dict_diff = {}
-        dict_a = {k.lower(): v for k, v in dict_a.items()}
-        dict_b = {k.lower(): v for k, v in dict_b.items()}
-        for k in dict_a:
-            if k in dict_b:
-                a_lowered = map(lambda x: x.lower(), dict_a[k])
-                b_lowered = map(lambda x: x.lower(), dict_b[k])
-                diff = list(set(a_lowered) - set(b_lowered))
-                if diff:
-                    dict_diff.update({k: diff})
-            else:
-                dict_diff.update({k: dict_a[k]})
-        return dict_diff
-
-    @contextmember
-    @staticmethod
-    def diff_of_two_dicts_no_lower(dict_a, dict_b):
-        """
-        Given two dictionaries:
-            dict_a: {'key_x': ['value_1', 'value_2'], 'key_y': ['value_3']}
-            dict_b: {'key_x': ['value_1'], 'key_z': ['value_4']}
-        Return the same dictionary representation of dict_a MINUS dict_b
-
-        No modifications to dict values.
+        Given two dictionaries of type Dict[str, List[str]]:
+            dict_a = {'key_x': ['value_1', 'VALUE_2'], 'KEY_Y': ['value_3']}
+            dict_b = {'key_x': ['value_1'], 'key_z': ['value_4']}
+        Return the same dictionary representation of dict_a MINUS dict_b,
+        performing a case-insensitive comparison between the strings in each.
+        All keys returned will be in the original case of dict_a.
+            returns {'key_x': ['VALUE_2'], 'KEY_Y': ['value_3']}
         """
 
         dict_diff = {}
+        dict_b_lowered = {k.casefold(): [x.casefold() for x in v] for k, v in dict_b.items()}
         for k in dict_a:
-            if k in dict_b:
-                a_lowered = map(lambda x: x.lower(), dict_a[k])
-                b_lowered = map(lambda x: x.lower(), dict_b[k])
-                diff = list(set(a_lowered) - set(b_lowered))
+            if k.casefold() in dict_b_lowered.keys():
+                diff = []
+                for v in dict_a[k]:
+                    if v.casefold() not in dict_b_lowered[k.casefold()]:
+                        diff.append(v)
                 if diff:
                     dict_diff.update({k: diff})
             else:

plugins/postgres/dbt/include/postgres/macros/adapters.sql

@@ -212,6 +212,6 @@
         and table_name = '{{ relation.identifier }}'
 {%- endmacro -%}
 
-{% macro postgres__are_grants_copied_over_when_replaced() %}
+{% macro postgres__copy_grants() %}
     {{ return(False) }}
 {% endmacro %}

tests/adapter/dbt/tests/adapter/grants/base_grants.py

@@ -16,14 +16,19 @@ def replace_all(text, dic):
 
 
 class BaseGrants:
-    def privilege_names(self):
-        # these privilege names are valid on most databases, but not all!
+    def privilege_grantee_name_overrides(self):
+        # these privilege and grantee names are valid on most databases, but not all!
         # looking at you, BigQuery
-        # optionally use this to map from select --> other_select_name, insert --> ...
-        return {"select": "select", "insert": "insert", "fake_privilege": "fake_privilege"}
-
-    def interpolate_privilege_names(self, yaml_text):
-        return replace_all(yaml_text, self.privilege_names())
+        # optionally use this to map from "select" --> "other_select_name", "insert" --> ...
+        return {
+            "select": "select",
+            "insert": "insert",
+            "fake_privilege": "fake_privilege",
+            "invalid_user": "invalid_user",
+        }
+
+    def interpolate_name_overrides(self, yaml_text):
+        return replace_all(yaml_text, self.privilege_grantee_name_overrides())
 
     @pytest.fixture(scope="class", autouse=True)
     def get_test_users(self, project):

tests/adapter/dbt/tests/adapter/grants/test_incremental_grants.py

@@ -37,7 +37,7 @@
 class BaseIncrementalGrants(BaseGrants):
     @pytest.fixture(scope="class")
     def models(self):
-        updated_schema = self.interpolate_privilege_names(incremental_model_schema_yml)
+        updated_schema = self.interpolate_name_overrides(incremental_model_schema_yml)
         return {
             "my_incremental_model.sql": my_incremental_model_sql,
             "schema.yml": updated_schema,
@@ -46,7 +46,7 @@ def models(self):
     def test_incremental_grants(self, project, get_test_users):
         # we want the test to fail, not silently skip
         test_users = get_test_users
-        select_privilege_name = self.privilege_names()["select"]
+        select_privilege_name = self.privilege_grantee_name_overrides()["select"]
         assert len(test_users) == 3
 
         # Incremental materialization, single select grant
@@ -67,7 +67,7 @@ def test_incremental_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_incremental_model", expected)
 
         # Incremental materialization, change select grant user
-        updated_yaml = self.interpolate_privilege_names(user2_incremental_model_schema_yml)
+        updated_yaml = self.interpolate_name_overrides(user2_incremental_model_schema_yml)
         write_file(updated_yaml, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"])
         assert len(results) == 1

tests/adapter/dbt/tests/adapter/grants/test_invalid_grants.py

@@ -31,29 +31,34 @@
 
 
 class BaseInvalidGrants(BaseGrants):
+    # The purpose of this test is to understand the user experience when providing
+    # an invalid 'grants' configuration. dbt will *not* try to intercept or interpret
+    # the database's own error at runtime -- it will just return those error messages.
+    # Hopefully they're helpful!
+
     @pytest.fixture(scope="class")
     def models(self):
         return {
             "my_invalid_model.sql": my_invalid_model_sql,
         }
 
-    # adapters will need to reimplement these methods with the specific
+    # Adapters will need to reimplement these methods with the specific
     # language of their database
     def grantee_does_not_exist_error(self):
         return "does not exist"
 
     def privilege_does_not_exist_error(self):
         return "unrecognized privilege"
 
-    def test_nonexistent_grantee(self, project, get_test_users, logs_dir):
+    def test_invalid_grants(self, project, get_test_users, logs_dir):
         # failure when grant to a user/role that doesn't exist
-        yaml_file = self.interpolate_privilege_names(invalid_user_table_model_schema_yml)
+        yaml_file = self.interpolate_name_overrides(invalid_user_table_model_schema_yml)
         write_file(yaml_file, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"], expect_pass=False)
         assert self.grantee_does_not_exist_error() in log_output
 
         # failure when grant to a privilege that doesn't exist
-        yaml_file = self.interpolate_privilege_names(invalid_privilege_table_model_schema_yml)
+        yaml_file = self.interpolate_name_overrides(invalid_privilege_table_model_schema_yml)
         write_file(yaml_file, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"], expect_pass=False)
         assert self.privilege_does_not_exist_error() in log_output

tests/adapter/dbt/tests/adapter/grants/test_model_grants.py

@@ -73,7 +73,7 @@
 class BaseModelGrants(BaseGrants):
     @pytest.fixture(scope="class")
     def models(self):
-        updated_schema = self.interpolate_privilege_names(model_schema_yml)
+        updated_schema = self.interpolate_name_overrides(model_schema_yml)
         return {
             "my_model.sql": my_model_sql,
             "schema.yml": updated_schema,
@@ -82,8 +82,8 @@ def models(self):
     def test_view_table_grants(self, project, get_test_users):
         # we want the test to fail, not silently skip
         test_users = get_test_users
-        select_privilege_name = self.privilege_names()["select"]
-        insert_privilege_name = self.privilege_names()["insert"]
+        select_privilege_name = self.privilege_grantee_name_overrides()["select"]
+        insert_privilege_name = self.privilege_grantee_name_overrides()["insert"]
         assert len(test_users) == 3
 
         # View materialization, single select grant
@@ -98,7 +98,7 @@ def test_view_table_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_model", expected)
 
         # View materialization, change select grant user
-        updated_yaml = self.interpolate_privilege_names(user2_model_schema_yml)
+        updated_yaml = self.interpolate_name_overrides(user2_model_schema_yml)
         write_file(updated_yaml, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"])
         assert len(results) == 1
@@ -107,7 +107,7 @@ def test_view_table_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_model", expected)
 
         # Table materialization, single select grant
-        updated_yaml = self.interpolate_privilege_names(table_model_schema_yml)
+        updated_yaml = self.interpolate_name_overrides(table_model_schema_yml)
         write_file(updated_yaml, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"])
         assert len(results) == 1
@@ -119,7 +119,7 @@ def test_view_table_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_model", expected)
 
         # Table materialization, change select grant user
-        updated_yaml = self.interpolate_privilege_names(user2_table_model_schema_yml)
+        updated_yaml = self.interpolate_name_overrides(user2_table_model_schema_yml)
         write_file(updated_yaml, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"])
         assert len(results) == 1
@@ -130,7 +130,7 @@ def test_view_table_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_model", expected)
 
         # Table materialization, multiple grantees
-        updated_yaml = self.interpolate_privilege_names(multiple_users_table_model_schema_yml)
+        updated_yaml = self.interpolate_name_overrides(multiple_users_table_model_schema_yml)
         write_file(updated_yaml, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"])
         assert len(results) == 1
@@ -141,7 +141,7 @@ def test_view_table_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_model", expected)
 
         # Table materialization, multiple privileges
-        updated_yaml = self.interpolate_privilege_names(multiple_privileges_table_model_schema_yml)
+        updated_yaml = self.interpolate_name_overrides(multiple_privileges_table_model_schema_yml)
         write_file(updated_yaml, project.project_root, "models", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "run"])
         assert len(results) == 1

tests/adapter/dbt/tests/adapter/grants/test_seed_grants.py

@@ -50,37 +50,46 @@
 
 
 class BaseSeedGrants(BaseGrants):
+    def seeds_support_partial_refresh(self):
+        return True
+
     @pytest.fixture(scope="class")
     def seeds(self):
-        updated_schema = self.interpolate_privilege_names(schema_base_yml)
+        updated_schema = self.interpolate_name_overrides(schema_base_yml)
         return {
             "my_seed.csv": seeds__my_seed_csv,
             "schema.yml": updated_schema,
         }
 
     def test_seed_grants(self, project, get_test_users):
         test_users = get_test_users
-        select_privilege_name = self.privilege_names()["select"]
+        select_privilege_name = self.privilege_grantee_name_overrides()["select"]
 
         # seed command
-        results = run_dbt(["seed"])
+        (results, log_output) = run_dbt_and_capture(["--debug", "seed"])
         assert len(results) == 1
         manifest = get_manifest(project.project_root)
         seed_id = "seed.test.my_seed"
         seed = manifest.nodes[seed_id]
         expected = {select_privilege_name: [test_users[0]]}
         assert seed.config.grants == expected
+        assert "grant " in log_output
         self.assert_expected_grants_match_actual(project, "my_seed", expected)
 
-        # run it again, nothing should have changed
+        # run it again, with no config changes
         (results, log_output) = run_dbt_and_capture(["--debug", "seed"])
         assert len(results) == 1
-        assert "revoke " not in log_output
-        assert "grant " not in log_output
+        if self.seeds_support_partial_refresh():
+            # grants carried over -- nothing should have changed
+            assert "revoke " not in log_output
+            assert "grant " not in log_output
+        else:
+            # seeds are always full-refreshed on this adapter, so we need to re-grant
+            assert "grant " in log_output
         self.assert_expected_grants_match_actual(project, "my_seed", expected)
 
         # change the grantee, assert it updates
-        updated_yaml = self.interpolate_privilege_names(user2_schema_base_yml)
+        updated_yaml = self.interpolate_name_overrides(user2_schema_base_yml)
         write_file(updated_yaml, project.project_root, "seeds", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "seed"])
         assert len(results) == 1
@@ -92,7 +101,7 @@ def test_seed_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_seed", expected)
 
         # change config to 'grants: {}' -- should be completely ignored
-        updated_yaml = self.interpolate_privilege_names(ignore_grants_yml)
+        updated_yaml = self.interpolate_name_overrides(ignore_grants_yml)
         write_file(updated_yaml, project.project_root, "seeds", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "seed"])
         assert len(results) == 1
@@ -104,15 +113,21 @@ def test_seed_grants(self, project, get_test_users):
         expected_config = {}
         expected_actual = {select_privilege_name: [test_users[1]]}
         assert seed.config.grants == expected_config
-        # ACTUAL grants will NOT match expected grants
-        self.assert_expected_grants_match_actual(project, "my_seed", expected_actual)
+        if self.seeds_support_partial_refresh():
+            # ACTUAL grants will NOT match expected grants
+            self.assert_expected_grants_match_actual(project, "my_seed", expected_actual)
+        else:
+            # there should be ZERO grants on the seed
+            self.assert_expected_grants_match_actual(project, "my_seed", expected_config)
 
         # now run with ZERO grants -- all grants should be removed
-        updated_yaml = self.interpolate_privilege_names(zero_grants_yml)
+        # whether explicitly (revoke) or implicitly (recreated without any grants added on)
+        updated_yaml = self.interpolate_name_overrides(zero_grants_yml)
         write_file(updated_yaml, project.project_root, "seeds", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "seed"])
         assert len(results) == 1
-        assert "revoke " in log_output
+        if self.seeds_support_partial_refresh():
+            assert "revoke " in log_output
         expected = {}
         self.assert_expected_grants_match_actual(project, "my_seed", expected)
 

tests/adapter/dbt/tests/adapter/grants/test_snapshot_grants.py

@@ -39,11 +39,14 @@
 class BaseSnapshotGrants(BaseGrants):
     @pytest.fixture(scope="class")
     def snapshots(self):
-        return {"my_snapshot.sql": my_snapshot_sql, "schema.yml": snapshot_schema_yml}
+        return {
+            "my_snapshot.sql": my_snapshot_sql,
+            "schema.yml": self.interpolate_name_overrides(snapshot_schema_yml),
+        }
 
     def test_snapshot_grants(self, project, get_test_users):
         test_users = get_test_users
-        select_privilege_name = self.privilege_names()["select"]
+        select_privilege_name = self.privilege_grantee_name_overrides()["select"]
 
         # run the snapshot
         results = run_dbt(["snapshot"])
@@ -63,7 +66,7 @@ def test_snapshot_grants(self, project, get_test_users):
         self.assert_expected_grants_match_actual(project, "my_snapshot", expected)
 
         # change the grantee, assert it updates
-        updated_yaml = self.interpolate_privilege_names(user2_snapshot_schema_yml)
+        updated_yaml = self.interpolate_name_overrides(user2_snapshot_schema_yml)
         write_file(updated_yaml, project.project_root, "snapshots", "schema.yml")
         (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"])
         assert len(results) == 1