add logging, add tests

dbt-labs · Jul 29, 2020 · bf7412c · bf7412c
1 parent 2ccf1da
commit bf7412c
Show file tree

Hide file tree

Showing 22 changed files with 244 additions and 2 deletions.
diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py
@@ -4,7 +4,7 @@
 import os
 from dataclasses import dataclass, field
 from datetime import datetime
-from itertools import chain
+from itertools import chain, islice
 from multiprocessing.synchronize import Lock
 from typing import (
     Dict, List, Optional, Union, Mapping, MutableMapping, Any, Set, Tuple,
@@ -1022,14 +1022,22 @@ def merge_from_artifact(
         Only non-ephemeral refable nodes are examined.
         """
         refables = set(NodeType.refable())
+        merged = set()
         for unique_id, node in other.nodes.items():
             if (
                 node.resource_type in refables and
                 not node.is_ephemeral and
                 unique_id not in selected
             ):
+                merged.add(unique_id)
                 self.nodes[unique_id] = node
 
+        # log up to 5 items
+        sample = list(islice(merged, 5))
+        logger.debug(
+            f'Merged {len(merged)} items from state (sample: {sample})'
+        )
+
 
 @dataclass
 class WritableManifest(JsonSchemaMixin, Writable, Readable):

diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py
@@ -89,4 +89,4 @@ def read(cls, path: str):
                 f'Could not read {cls.__name__} at "{path}" as JSON: {exc}'
             ) from exc
 
-        return cls.from_dict(data)
+        return cls.from_dict(data)  # type: ignore
diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py
@@ -966,6 +966,7 @@ def expected_seeded_manifest(self, model_database=None):
                     'config': model_config,
                     'schema': my_schema_name,
                     'database': model_database,
+                    'deferred': False,
                     'alias': 'model',
                     'description': 'The test model',
                     'columns': {
@@ -1115,6 +1116,7 @@ def expected_seeded_manifest(self, model_database=None):
                     'schema': my_schema_name,
                     'database': self.default_database,
                     'alias': 'seed',
+                    'deferred': False,
                     'description': 'The test seed',
                     'columns': {
                         'id': {
@@ -1183,6 +1185,7 @@ def expected_seeded_manifest(self, model_database=None):
                         'macros': ['macro.dbt.test_not_null'],
                         'nodes': ['model.test.model'],
                     },
+                    'deferred': False,
                     'description': '',
                     'fqn': ['test', 'schema_test', 'not_null_model_id'],
                     'name': 'not_null_model_id',
@@ -1237,6 +1240,7 @@ def expected_seeded_manifest(self, model_database=None):
                         'macros': ['macro.test.test_nothing'],
                         'nodes': ['model.test.model'],
                     },
+                    'deferred': False,
                     'description': '',
                     'fqn': ['test', 'schema_test', 'test_nothing_model_'],
                     'name': 'test_nothing_model_',
@@ -1290,6 +1294,7 @@ def expected_seeded_manifest(self, model_database=None):
                         'macros': ['macro.dbt.test_unique'],
                         'nodes': ['model.test.model'],
                     },
+                    'deferred': False,
                     'description': '',
                     'fqn': ['test', 'schema_test', 'unique_model_id'],
                     'name': 'unique_model_id',
@@ -1386,6 +1391,7 @@ def expected_postgres_references_manifest(self, model_database=None):
                         'macros': [],
                         'nodes': ['source.test.my_source.my_table']
                     },
+                    'deferred': False,
                     'description': '',
                     'docs': {'show': True},
                     'fqn': ['test', 'ephemeral_copy'],
@@ -1448,6 +1454,7 @@ def expected_postgres_references_manifest(self, model_database=None):
                         'macros': [],
                         'nodes': ['model.test.ephemeral_copy']
                     },
+                    'deferred': False,
                     'description': 'A summmary table of the ephemeral copy of the seed data',
                     'docs': {'show': True},
                     'fqn': ['test', 'ephemeral_summary'],
@@ -1512,6 +1519,7 @@ def expected_postgres_references_manifest(self, model_database=None):
                         'macros': [],
                         'nodes': ['model.test.ephemeral_summary']
                     },
+                    'deferred': False,
                     'description': 'A view of the summary of the ephemeral copy of the seed data',
                     'docs': {'show': True},
                     'fqn': ['test', 'view_summary'],
@@ -1594,6 +1602,7 @@ def expected_postgres_references_manifest(self, model_database=None):
                     },
                     'sources': [],
                     'depends_on': {'macros': [], 'nodes': []},
+                    'deferred': False,
                     'description': 'The test seed',
                     'docs': {'show': True},
                     'fqn': ['test', 'seed'],
@@ -1879,6 +1888,7 @@ def expected_bigquery_complex_manifest(self):
                             'tags': [],
                         },
                     },
+                    'deferred': False,
                     'description': 'A clustered and partitioned copy of the test model',
                     'patch_path': self.dir('bq_models/schema.yml'),
                     'docs': {'show': True},
@@ -1958,6 +1968,7 @@ def expected_bigquery_complex_manifest(self):
                             'tags': [],
                         },
                     },
+                    'deferred': False,
                     'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns',
                     'patch_path': self.dir('bq_models/schema.yml'),
                     'docs': {'show': True},
@@ -2038,6 +2049,7 @@ def expected_bigquery_complex_manifest(self):
                             'tags': [],
                         },
                     },
+                    'deferred': False,
                     'description': 'The test model',
                     'patch_path': self.dir('bq_models/schema.yml'),
                     'docs': {'show': True},
@@ -2083,6 +2095,7 @@ def expected_bigquery_complex_manifest(self):
                     'meta': {},
                     'unique_id': 'model.test.nested_table',
                     'columns': {},
+                    'deferred': False,
                     'description': '',
                     'docs': {'show': True},
                     'compiled': True,
@@ -2164,6 +2177,7 @@ def expected_bigquery_complex_manifest(self):
                             'tags': [],
                         },
                     },
+                    'deferred': False,
                     'description': 'The test seed',
                     'docs': {'show': True},
                     'compiled': True,
@@ -2268,6 +2282,7 @@ def expected_redshift_incremental_view_manifest(self):
                     'schema': my_schema_name,
                     'database': self.default_database,
                     'alias': 'model',
+                    'deferred': False,
                     'description': 'The test model',
                     'columns': {
                         'id': {
@@ -2387,6 +2402,7 @@ def expected_redshift_incremental_view_manifest(self):
                             'tags': [],
                         },
                     },
+                    'deferred': False,
                     'description': 'The test seed',
                     'docs': {'show': True},
                     'compiled': True,
@@ -2559,6 +2575,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
                         'macros': [],
                         'nodes': ['seed.test.seed']
                     },
+                    'deferred': False,
                     'description': 'The test model',
                     'docs': {'show': False},
                     'extra_ctes': [],
@@ -2728,6 +2745,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
                     },
                     'sources': [],
                     'depends_on': {'macros': [], 'nodes': []},
+                    'deferred': False,
                     'description': 'The test seed',
                     'docs': {'show': True},
                     'extra_ctes': [],
@@ -2784,6 +2802,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
                         'macros': ['macro.dbt.test_not_null'],
                         'nodes': ['model.test.model'],
                     },
+                    'deferred': False,
                     'description': '',
                     'docs': {'show': True},
                     'extra_ctes': [],
@@ -2848,6 +2867,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
                         'macros': ['macro.test.test_nothing'],
                         'nodes': ['model.test.model'],
                     },
+                    'deferred': False,
                     'description': '',
                     'docs': {'show': True},
                     'extra_ctes': [],
@@ -2911,6 +2931,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
                         'macros': ['macro.dbt.test_unique'],
                         'nodes': ['model.test.model'],
                     },
+                    'deferred': False,
                     'description': '',
                     'docs': {'show': True},
                     'extra_ctes': [],
@@ -3016,6 +3037,7 @@ def expected_postgres_references_run_results(self):
                         'nodes': ['model.test.ephemeral_copy'],
                         'macros': []
                     },
+                    'deferred': False,
                     'description': (
                         'A summmary table of the ephemeral copy of the seed data'
                     ),
@@ -3097,6 +3119,7 @@ def expected_postgres_references_run_results(self):
                         'nodes': ['model.test.ephemeral_summary'],
                         'macros': []
                     },
+                    'deferred': False,
                     'description': (
                         'A view of the summary of the ephemeral copy of the '
                         'seed data'
@@ -3192,6 +3215,7 @@ def expected_postgres_references_run_results(self):
                     },
                     'sources': [],
                     'depends_on': {'macros': [], 'nodes': []},
+                    'deferred': False,
                     'description': 'The test seed',
                     'docs': {'show': True},
                     'extra_ctes': [],

diff --git a/test/integration/062_defer_state_test/changed_models/ephemeral_model.sql b/test/integration/062_defer_state_test/changed_models/ephemeral_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='ephemeral') }}
+select * from {{ ref('view_model') }}
diff --git a/test/integration/062_defer_state_test/changed_models/schema.yml b/test/integration/062_defer_state_test/changed_models/schema.yml
@@ -0,0 +1,9 @@
+version: 2
+models:
+  - name: view_model
+    columns:
+      - name: id
+        tests:
+          - unique
+          - not_null
+      - name: name
diff --git a/test/integration/062_defer_state_test/changed_models/table_model.sql b/test/integration/062_defer_state_test/changed_models/table_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='table') }}
+select * from {{ ref('ephemeral_model') }}
diff --git a/test/integration/062_defer_state_test/changed_models/view_model.sql b/test/integration/062_defer_state_test/changed_models/view_model.sql
@@ -0,0 +1 @@
+select * from no.such.table
diff --git a/test/integration/062_defer_state_test/changed_models_bad/ephemeral_model.sql b/test/integration/062_defer_state_test/changed_models_bad/ephemeral_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='ephemeral') }}
+select * from no.such.table
diff --git a/test/integration/062_defer_state_test/changed_models_bad/schema.yml b/test/integration/062_defer_state_test/changed_models_bad/schema.yml
@@ -0,0 +1,9 @@
+version: 2
+models:
+  - name: view_model
+    columns:
+      - name: id
+        tests:
+          - unique
+          - not_null
+      - name: name
diff --git a/test/integration/062_defer_state_test/changed_models_bad/table_model.sql b/test/integration/062_defer_state_test/changed_models_bad/table_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='table') }}
+select * from {{ ref('ephemeral_model') }}
diff --git a/test/integration/062_defer_state_test/changed_models_bad/view_model.sql b/test/integration/062_defer_state_test/changed_models_bad/view_model.sql
@@ -0,0 +1 @@
+select * from no.such.table
diff --git a/test/integration/062_defer_state_test/data/seed.csv b/test/integration/062_defer_state_test/data/seed.csv
@@ -0,0 +1,3 @@
+id,name
+1,Alice
+2,Bob
diff --git a/test/integration/062_defer_state_test/models/ephemeral_model.sql b/test/integration/062_defer_state_test/models/ephemeral_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='ephemeral') }}
+select * from {{ ref('view_model') }}
diff --git a/test/integration/062_defer_state_test/models/schema.yml b/test/integration/062_defer_state_test/models/schema.yml
@@ -0,0 +1,9 @@
+version: 2
+models:
+  - name: view_model
+    columns:
+      - name: id
+        tests:
+          - unique
+          - not_null
+      - name: name
diff --git a/test/integration/062_defer_state_test/models/table_model.sql b/test/integration/062_defer_state_test/models/table_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='table') }}
+select * from {{ ref('ephemeral_model') }}
diff --git a/test/integration/062_defer_state_test/models/view_model.sql b/test/integration/062_defer_state_test/models/view_model.sql
@@ -0,0 +1 @@
+select * from {{ ref('seed') }}
diff --git a/test/integration/062_defer_state_test/snapshots/my_snapshot.sql b/test/integration/062_defer_state_test/snapshots/my_snapshot.sql
@@ -0,0 +1,14 @@
+{% snapshot my_cool_snapshot %}
+
+    {{
+        config(
+            target_database=database,
+            target_schema=schema,
+            unique_key='id',
+            strategy='check',
+            check_cols=['id'],
+        )
+    }}
+    select * from {{ ref('view_model') }}
+
+{% endsnapshot %}