Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Redshift catalog generation (#831) #866

Merged
merged 7 commits into from
Jul 24, 2018
Merged

Redshift catalog generation (#831) #866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6bf9c32
Add some probably not-functional redshift support
jwerderits Jul 16, 2018
9829555
view -> LATE BINDING VIEW
jwerderits Jul 17, 2018
9afe77a
Add a partially functioning test for late binding views
Jul 24, 2018
eed4b5c
When generating the catalog, explicitly clear the transaction we crea…
jwerderits Jul 18, 2018
bf67477
Fix docs generate tests
Jul 24, 2018
526a449
Cleanup
jwerderits Jul 18, 2018
9655869
tests lost in the rebase :(
Jul 24, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions dbt/adapters/default/impl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -812,9 +812,12 @@ def run_operation(cls, profile, project_cfg, manifest, operation_name,
###
@classmethod
def get_catalog(cls, profile, project_cfg, manifest):
results = cls.run_operation(profile, project_cfg, manifest,
GET_CATALOG_OPERATION_NAME,
GET_CATALOG_RESULT_KEY)
try:
results = cls.run_operation(profile, project_cfg, manifest,
GET_CATALOG_OPERATION_NAME,
GET_CATALOG_RESULT_KEY)
finally:
cls.release_connection(profile, GET_CATALOG_OPERATION_NAME)
schemas = list({
node.to_dict()['schema']
for node in manifest.nodes.values()
Expand Down
61 changes: 61 additions & 0 deletions dbt/include/global_project/macros/adapters/common.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,3 +162,64 @@
order by "column_index"
{%- endcall -%}
{%- endmacro %}


{% macro redshift__get_catalog() -%}
{%- call statement('catalog', fetch_result=True) -%}
with late_binding as (
select
table_schema,
table_name,
'LATE BINDING VIEW' as table_type,
null as table_comment,

column_name,
column_index,
column_type,
null as column_comment
from pg_get_late_binding_view_cols()
cols(table_schema name, table_name name, column_name name,
column_type varchar,
column_index int)
order by "column_index"
),
tables as (
select
table_schema,
table_name,
table_type

from information_schema.tables

),

columns as (

select
table_schema,
table_name,
null as table_comment,

column_name,
ordinal_position as column_index,
data_type as column_type,
null as column_comment


from information_schema.columns

)

select *
from tables
join columns using (table_schema, table_name)

where table_schema != 'information_schema'
and table_schema not like 'pg_%'

union all

select * from late_binding
order by "column_index"
{%- endcall -%}
{%- endmacro %}
7 changes: 7 additions & 0 deletions test/integration/029_docs_generate_tests/rs_models/model.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{{
config(
materialized='view', bind=False
)
}}

select * from {{ ref('seed') }}
230 changes: 227 additions & 3 deletions test/integration/029_docs_generate_tests/test_docs_generate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,11 @@ def project_config(self):
}
}

def run_and_generate(self):
self.use_default_project({"data-paths": [self.dir("seed")]})
def run_and_generate(self, extra=None):
project = {"data-paths": [self.dir("seed")]}
if extra:
project.update(extra)
self.use_default_project(project)

self.assertEqual(len(self.run_dbt(["seed"])), 1)
self.assertEqual(len(self.run_dbt()), 1)
Expand Down Expand Up @@ -122,7 +125,8 @@ def expected_seeded_manifest(self):
'resource_type': 'seed',
'raw_sql': '-- csv --',
'package_name': 'test',
'original_file_path': self.dir('seed/seed.csv'),
'original_file_path': self.dir(os.path.join('seed',
'seed.csv')),
'refs': [],
'depends_on': {'nodes': [], 'macros': []},
'unique_id': 'seed.test.seed',
Expand Down Expand Up @@ -479,3 +483,223 @@ def test__bigquery__nested_models(self):
},
}
self.verify_manifest(expected_manifest)

@use_profile('redshift')
def test__redshift__run_and_generate(self):
self.run_and_generate()
my_schema_name = self.unique_schema()
expected_cols = [
{
'name': 'id',
'index': 1,
'type': 'integer',
'comment': None,
},
{
'name': 'first_name',
'index': 2,
'type': 'character varying',
'comment': None,
},
{
'name': 'email',
'index': 3,
'type': 'character varying',
'comment': None,
},
{
'name': 'ip_address',
'index': 4,
'type': 'character varying',
'comment': None,
},
{
'name': 'updated_at',
'index': 5,
'type': 'timestamp without time zone',
'comment': None,
},
]
expected_catalog = {
'model': {
'metadata': {
'schema': my_schema_name,
'name': 'model',
'type': 'VIEW',
'comment': None,
},
'columns': expected_cols,
},
'seed': {
'metadata': {
'schema': my_schema_name,
'name': 'seed',
'type': 'BASE TABLE',
'comment': None,
},
'columns': expected_cols,
},
}
self.verify_catalog(expected_catalog)
self.verify_manifest(self.expected_seeded_manifest())

@use_profile('redshift')
def test__redshift__incremental_view(self):
self.run_and_generate({'source-paths': [self.dir('rs_models')]})
my_schema_name = self.unique_schema()
expected_catalog = {
'model': {
'metadata': {
'schema': my_schema_name,
'name': 'model',
'type': 'LATE BINDING VIEW',
'comment': None,
},
'columns': [
{
'name': 'id',
'index': 1,
'type': 'integer',
'comment': None,
},
{
'name': 'first_name',
'index': 2,
'type': 'character varying(5)',
'comment': None,
},
{
'name': 'email',
'index': 3,
'type': 'character varying(23)',
'comment': None,
},
{
'name': 'ip_address',
'index': 4,
'type': 'character varying(14)',
'comment': None,
},
{
'name': 'updated_at',
'index': 5,
'type': 'timestamp without time zone',
'comment': None,
},
],
},
'seed': {
'metadata': {
'schema': my_schema_name,
'name': 'seed',
'type': 'BASE TABLE',
'comment': None,
},
'columns': [
{
'name': 'id',
'index': 1,
'type': 'integer',
'comment': None,
},
{
'name': 'first_name',
'index': 2,
'type': 'character varying',
'comment': None,
},
{
'name': 'email',
'index': 3,
'type': 'character varying',
'comment': None,
},
{
'name': 'ip_address',
'index': 4,
'type': 'character varying',
'comment': None,
},
{
'name': 'updated_at',
'index': 5,
'type': 'timestamp without time zone',
'comment': None,
},
],
},
}
self.verify_catalog(expected_catalog)
model_sql_path = self.dir('rs_models/model.sql')
expected_manifest = {
"nodes": {
"model.test.model": {
"name": "model",
"root_path": os.getcwd(),
"resource_type": "model",
"path": "model.sql",
"original_file_path": model_sql_path,
"package_name": "test",
"raw_sql": open(model_sql_path).read().rstrip('\n'),
"refs": [["seed"]],
"depends_on": {
"nodes": ["seed.test.seed"],
"macros": [],
},
"unique_id": "model.test.model",
"empty": False,
"fqn": ["test", "model"],
"tags": [],
"config": {
"bind": False,
"enabled": True,
"materialized": "view",
"pre-hook": [],
"post-hook": [],
"vars": {},
"column_types": {},
"quoting": {},
},
"schema": my_schema_name,
"alias": "model"
},
"seed.test.seed": {
"path": "seed.csv",
"name": "seed",
"root_path": os.getcwd(),
"resource_type": "seed",
"raw_sql": "-- csv --",
"package_name": "test",
"original_file_path": self.dir("seed/seed.csv"),
"refs": [],
"depends_on": {
"nodes": [],
"macros": [],
},
"unique_id": "seed.test.seed",
"empty": False,
"fqn": ["test", "seed"],
"tags": [],
"config": {
"enabled": True,
"materialized": "seed",
"pre-hook": [],
"post-hook": [],
"vars": {},
"column_types": {},
"quoting": {},
},
"schema": my_schema_name,
"alias": "seed"
},
},
"parent_map": {
"model.test.model": ["seed.test.seed"],
"seed.test.seed": []
},
"child_map": {
"model.test.model": [],
"seed.test.seed": ["model.test.model"]
}
}
self.verify_manifest(expected_manifest)