Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle "meta:" key in schema/sources.yml #2015

Merged
6 changes: 6 additions & 0 deletions core/dbt/contracts/graph/parsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ def __len__(self):
class ColumnInfo(JsonSchemaMixin, Replaceable):
name: str
description: str = ''
meta: Dict[str, Any] = field(default_factory=dict)
data_type: Optional[str] = None


Expand Down Expand Up @@ -180,6 +181,7 @@ def patch(self, patch):
self.description = patch.description
self.columns = patch.columns
self.docrefs = patch.docrefs
self.meta = patch.meta
if dbt.flags.STRICT_MODE:
self.to_dict(validate=True)

Expand Down Expand Up @@ -215,6 +217,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
docrefs: List[Docref] = field(default_factory=list)
description: str = field(default='')
columns: Dict[str, ColumnInfo] = field(default_factory=dict)
meta: Dict[str, Any] = field(default_factory=dict)
patch_path: Optional[str] = None
build_path: Optional[str] = None

Expand Down Expand Up @@ -455,6 +458,7 @@ class ParsedNodePatch(JsonSchemaMixin, Replaceable):
original_file_path: str
columns: Dict[str, ColumnInfo]
docrefs: List[Docref]
meta: Dict[str, Any]


@dataclass
Expand Down Expand Up @@ -507,6 +511,8 @@ class ParsedSourceDefinition(
docrefs: List[Docref] = field(default_factory=list)
description: str = ''
columns: Dict[str, ColumnInfo] = field(default_factory=dict)
meta: Dict[str, Any] = field(default_factory=dict)
source_meta: Dict[str, Any] = field(default_factory=dict)

@property
def is_ephemeral_model(self):
Expand Down
3 changes: 3 additions & 0 deletions core/dbt/contracts/graph/unparsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ class UnparsedRunHook(UnparsedNode):
class NamedTested(JsonSchemaMixin, Replaceable):
name: str
description: str = ''
meta: Dict[str, Any] = field(default_factory=dict)
data_type: Optional[str] = None
tests: Optional[List[Union[Dict[str, Any], str]]] = None

Expand Down Expand Up @@ -159,6 +160,7 @@ class ExternalPartition(AdditionalPropertiesAllowed, Replaceable):
name: str = ''
description: str = ''
data_type: str = ''
meta: Dict[str, Any] = field(default_factory=dict)

def __post_init__(self):
if self.name == '' or self.data_type == '':
Expand Down Expand Up @@ -206,6 +208,7 @@ def __post_init__(self):
class UnparsedSourceDefinition(JsonSchemaMixin, Replaceable):
name: str
description: str = ''
meta: Dict[str, Any] = field(default_factory=dict)
database: Optional[str] = None
schema: Optional[str] = None
loader: str = ''
Expand Down
15 changes: 11 additions & 4 deletions core/dbt/parser/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,11 @@ def __init__(self):
self.column_info: Dict[str, ColumnInfo] = {}
self.docrefs: List[Docref] = []

def add(self, column_name, description, data_type):
def add(self, column_name, description, data_type, meta):
self.column_info[column_name] = ColumnInfo(name=column_name,
description=description,
data_type=data_type)
data_type=data_type,
meta=meta)


def collect_docrefs(
Expand Down Expand Up @@ -222,9 +223,10 @@ def parse_column(
column_name = column.name
description = column.description
data_type = column.data_type
meta = column.meta
collect_docrefs(block.target, refs, column_name, description)

refs.add(column_name, description, data_type)
refs.add(column_name, description, data_type, meta)

if not column.tests:
return
Expand Down Expand Up @@ -340,6 +342,7 @@ def generate_source_node(
NodeType.Source, self.project.project_name, source.name, table.name
])
description = table.description or ''
meta = table.meta or {}
source_description = source.description or ''
collect_docrefs(source, refs, None, description, source_description)

Expand All @@ -348,6 +351,7 @@ def generate_source_node(
freshness = self._calculate_freshness(source, table)
quoting = source.quoting.merged(table.quoting)
path = block.path.original_file_path
source_meta = source.meta or {}

return ParsedSourceDefinition(
package_name=self.project.project_name,
Expand All @@ -364,6 +368,8 @@ def generate_source_node(
external=table.external,
source_name=source.name,
source_description=source_description,
source_meta=source_meta,
meta=meta,
loader=source.loader,
docrefs=refs.docrefs,
loaded_at_field=loaded_at_field,
Expand All @@ -385,7 +391,8 @@ def generate_node_patch(
original_file_path=block.path.original_file_path,
description=description,
columns=refs.column_info,
docrefs=refs.docrefs
docrefs=refs.docrefs,
meta=block.target.meta,
)

def parse_target_model(
Expand Down
Loading