diff --git a/CHANGELOG.md b/CHANGELOG.md index 0021a9f9ff4..000c3a4e476 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## dbt 0.19.0 (Release TBD) ### Breaking changes -- The format for sources.json, run-results.json, manifest.json, and catalog.json has changed to include a common metadata field ([#2761](https://github.com/fishtown-analytics/dbt/issues/2761), [#2778](https://github.com/fishtown-analytics/dbt/pull/2778), [#2763](https://github.com/fishtown-analytics/dbt/issues/2763), [#2784](https://github.com/fishtown-analytics/dbt/pull/2784)) +- The format for sources.json, run-results.json, manifest.json, and catalog.json has changed to include a common metadata field ([#2761](https://github.com/fishtown-analytics/dbt/issues/2761), [#2778](https://github.com/fishtown-analytics/dbt/pull/2778), [#2763](https://github.com/fishtown-analytics/dbt/issues/2763), [#2784](https://github.com/fishtown-analytics/dbt/pull/2784), [#2764](https://github.com/fishtown-analytics/dbt/issues/2764), [#2785](https://github.com/fishtown-analytics/dbt/pull/2785)) ### Features - dbt will compare configurations using the un-rendered form of the config block in dbt_project.yml ([#2713](https://github.com/fishtown-analytics/dbt/issues/2713), [#2735](https://github.com/fishtown-analytics/dbt/pull/2735)) diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py index 7a1ff57f888..817ac3e264b 100644 --- a/core/dbt/contracts/util.py +++ b/core/dbt/contracts/util.py @@ -1,4 +1,5 @@ import dataclasses +import os from datetime import datetime from typing import ( List, Tuple, ClassVar, Type, TypeVar, Dict, Any, Optional @@ -123,6 +124,16 @@ def __str__(self) -> str: SCHEMA_VERSION_KEY = 'dbt_schema_version' +METADATA_ENV_PREFIX = 'DBT_ENV_CUSTOM_ENV_' + + +def get_metadata_env() -> Dict[str, str]: + return { + k[len(METADATA_ENV_PREFIX):]: v for k, v in os.environ.items() + if k.startswith(METADATA_ENV_PREFIX) + } + + @dataclasses.dataclass class BaseArtifactMetadata(JsonSchemaMixin): dbt_schema_version: str @@ -133,6 +144,7 @@ class BaseArtifactMetadata(JsonSchemaMixin): invocation_id: Optional[str] = dataclasses.field( default_factory=get_invocation_id ) + env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_env) def schema_version(name: str, version: int): diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index fc48bb5a8c8..21666dd5af2 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -5,7 +5,6 @@ import shutil import tempfile import time -import uuid from datetime import datetime from unittest.mock import ANY, patch @@ -112,6 +111,12 @@ def setUp(self): self.alternate_schema = self.alternate_schema.upper() self._created_schemas.add(self.alternate_schema) + os.environ['DBT_ENV_CUSTOM_ENV_env_key'] = 'env_value' + + def tearDown(self): + super().tearDown() + del os.environ['DBT_ENV_CUSTOM_ENV_env_key'] + @property def schema(self): @@ -2563,6 +2568,12 @@ def verify_metadata(self, metadata, dbt_schema_version): assert 'dbt_schema_version' in metadata assert metadata['dbt_schema_version'] == dbt_schema_version assert metadata['invocation_id'] == dbt.tracking.active_user.invocation_id + key = 'env_key' + if os.name == 'nt': + key = key.upper() + assert metadata['env'] == { + key: 'env_value' + } def verify_manifest(self, expected_manifest): self.assertTrue(os.path.exists('./target/manifest.json')) diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index f273cf0f903..250079f87ae 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -56,6 +56,11 @@ def setUp(self): self._id = 101 # this is the db initial value self.last_inserted_time = "2016-09-19T14:45:51+00:00" + os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value' + + def tearDown(self): + super().tearDown() + del os.environ['DBT_ENV_CUSTOM_ENV_key'] def _set_updated_at_to(self, delta): insert_time = datetime.utcnow() + delta @@ -244,6 +249,10 @@ def _assert_freshness_results(self, path, state): assert data['metadata']['dbt_schema_version'] == 'https://schemas.getdbt.com/dbt/sources/v1.json' assert data['metadata']['dbt_version'] == dbt.version.__version__ assert data['metadata']['invocation_id'] == dbt.tracking.active_user.invocation_id + key = 'key' + if os.name == 'nt': + key = key.upper() + assert data['metadata']['env'] == {key: 'value'} last_inserted_time = self.last_inserted_time diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index de456823588..87d474a1cda 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -1,3 +1,4 @@ +import os import unittest from unittest import mock @@ -41,6 +42,9 @@ }) +ENV_KEY_NAME = 'KEY' if os.name == 'nt' else 'key' + + class ManifestTest(unittest.TestCase): def setUp(self): dbt.flags.STRICT_MODE = True @@ -212,6 +216,11 @@ def setUp(self): for source in self.sources.values(): source.validate(source.to_dict()) + os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value' + + def tearDown(self): + del os.environ['DBT_ENV_CUSTOM_ENV_key'] + @freezegun.freeze_time('2018-02-14T09:15:13Z') def test__no_nodes(self): manifest = Manifest( @@ -231,6 +240,7 @@ def test__no_nodes(self): 'generated_at': '2018-02-14T09:15:13Z', 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', 'dbt_version': dbt.version.__version__, + 'env': {ENV_KEY_NAME: 'value'}, # invocation_id is None, so it will not be present }, 'docs': {}, @@ -376,6 +386,7 @@ def test_no_nodes_with_metadata(self, mock_user): 'send_anonymous_usage_stats': False, 'adapter_type': 'postgres', 'invocation_id': '01234567-0123-0123-0123-0123456789ab', + 'env': {ENV_KEY_NAME: 'value'}, }, 'disabled': [], } @@ -586,6 +597,10 @@ def setUp(self): checksum=FileHash.empty(), ), } + os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value' + + def tearDown(self): + del os.environ['DBT_ENV_CUSTOM_ENV_key'] @freezegun.freeze_time('2018-02-14T09:15:13Z') def test__no_nodes(self): @@ -607,6 +622,7 @@ def test__no_nodes(self): 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', 'dbt_version': dbt.version.__version__, 'invocation_id': '01234567-0123-0123-0123-0123456789ab', + 'env': {ENV_KEY_NAME: 'value'}, }, 'docs': {}, 'disabled': [],